Topic: Bitcoin based Blockchain compression algorithm - page 2. (Read 3700 times)

FindBlockPos bool (&state CValidationState, &pos CDiskBlockPos, int nAddSize unsigned, int nHeight unsigned, nTime Uint64, fKnown bool = false)
{
...
        /* while (infoLastBlockFile.nSize + nAddSize >= MAX_BLOCKFILE_SIZE) { */
        if( ((nHeight / 10000) > 0) && ((nHeight % 10000) == 0) ) {
            printf("nHeight = [%d], Leaving block file %i: %s\n", nHeight, nLastBlockFile, infoLastBlockFile.ToString().c_str());
            FlushBlockFile(true);
            nLastBlockFile++;
            infoLastBlockFile.SetNull();
            pblocktree->ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile); // check whether data for the new file somehow already exist; can fail just fine
            fUpdatedLast = true;
        }
...
}

#include "lz4/lz4.h"
#include "lzma/LzmaLib.h"

int StreamToBuffer(CDataStream &ds, string& sRzt, int iSaveBufSize)
{
int bsz = ds.size();
int iRsz = bsz;
if( iSaveBufSize > 0 ){ iRsz = iRsz + 4; }
sRzt.resize(iRsz);
char* ppp = (char*)sRzt.c_str();
if( iSaveBufSize > 0 ){ ppp = ppp + 4; }
ds.read(ppp, bsz);
if( iSaveBufSize > 0 ){ *(unsigned int *)(ppp - 4) = bsz; }
return iRsz;
}

int CBlockToBuffer(CBlock *pb, string& sRzt)
{
CDataStream ssBlock(SER_DISK, CLIENT_VERSION);
ssBlock << (*pb);
int bsz = StreamToBuffer(ssBlock, sRzt, 0);
return bsz;
}

int writeBufToFile(char* pBuf, int bufLen, string fName)
{
int rzt = 0;
std::ofstream oFs(fName.c_str(), std::ios::out | std::ofstream::binary);
if( oFs.is_open() )
{
if( pBuf ) oFs.write(pBuf, bufLen);
oFs.close();
rzt++;
}
return rzt;
}

int lz4_pack_buf(char* pBuf, int bufLen, string& sRzt)
{
int worstCase = 0;
int lenComp = 0;
    try{
worstCase = LZ4_compressBound( bufLen );
//std::vector vchCompressed;   //vchCompressed.resize(worstCase);
sRzt.resize(worstCase + 4);
char* pp = (char *)sRzt.c_str();
lenComp = LZ4_compress(pBuf, pp + 4, bufLen);
if( lenComp > 0 ){ *(unsigned int *)pp = bufLen;   lenComp = lenComp + 4; }
}
    catch (std::exception &e) {
        printf("lz4_pack_buf err [%s]:: buf len %d, worstCase[%d], lenComp[%d] \n", e.what(), bufLen, worstCase, lenComp);
    }
return lenComp;
}

int lz4_unpack_buf(const char* pZipBuf, unsigned int zipLen, string& sRzt)
{
int rzt = 0;
unsigned int realSz = *(unsigned int *)pZipBuf;
if( fDebug )printf("lz4_unpack_buf:: zipLen [%d], realSz [%d],  \n", zipLen, realSz);
sRzt.resize(realSz);
char* pOutData = (char*)sRzt.c_str();

    // -- decompress
rzt = LZ4_decompress_safe(pZipBuf + 4, pOutData, zipLen, realSz);
    if ( rzt != (int) realSz)
    {
            if( fDebug )printf("lz4_unpack_buf:: Could not decompress message data. [%d :: %d] \n", rzt, realSz);
            sRzt.resize(0);
    }
return rzt;
}

int CBlockFromBuffer(CBlock* block, char* pBuf, int bufLen)
{
CDataStream ssBlock(SER_DISK, CLIENT_VERSION);
ssBlock.write(pBuf, bufLen);   int i = ssBlock.size();
ssBlock >> (*block);
return i;
}

int lz4_pack_block(CBlock* block, string& sRzt)
{
int rzt = 0;
string sbf;
int bsz = CBlockToBuffer(block, sbf);
if( bsz > 12 )
{
char* pBuf = (char*)sbf.c_str();
rzt = lz4_pack_buf(pBuf, bsz, sRzt);
//if( lzRzt > 0 ){ rzt = lzRzt; }  // + 4; }
}
sbf.resize(0);
return rzt;
}

int lzma_depack_buf(unsigned char* pLzmaBuf, int bufLen, string& sRzt)
{
int rzt = 0;
unsigned int dstLen = *(unsigned int *)pLzmaBuf;
    sRzt.resize(dstLen);
unsigned char* pOutBuf = (unsigned char*)sRzt.c_str();
    unsigned srcLen = bufLen - LZMA_PROPS_SIZE - 4;
SRes res = LzmaUncompress(pOutBuf, &dstLen, &pLzmaBuf[LZMA_PROPS_SIZE + 4], &srcLen, &pLzmaBuf[4], LZMA_PROPS_SIZE);
if( res == SZ_OK )//assert(res == SZ_OK);
{
//outBuf.resize(dstLen); // If uncompressed data can be smaller
rzt = dstLen;
}else sRzt.resize(0);
if( fDebug ) printf("lzma_depack_buf:: res [%d], dstLen[%d],  rzt = [%d]\n", res, dstLen, rzt);
return rzt;
}

int lzma_pack_buf(unsigned char* pBuf, int bufLen, string& sRzt, int iLevel, unsigned int iDictSize)  // (1 << 17) = 131072 = 128K
{
int res = 0;
int rzt = 0;
unsigned propsSize = LZMA_PROPS_SIZE;
unsigned destLen = bufLen + (bufLen / 3) + 128;
    try{
sRzt.resize(propsSize + destLen + 4);
unsigned char* pOutBuf = (unsigned char*)sRzt.c_str();

res = LzmaCompress(&pOutBuf[LZMA_PROPS_SIZE + 4], &destLen, pBuf, bufLen, &pOutBuf[4], &propsSize,
                                      iLevel, iDictSize, -1, -1, -1, -1, -1);  // 1 << 14 = 16K, 1 << 16 = 64K
  
//assert(propsSize == LZMA_PROPS_SIZE);
//assert(res == SZ_OK);
if( (res == SZ_OK) && (propsSize == LZMA_PROPS_SIZE) )  
{
//outBuf.resize(propsSize + destLen);
*(unsigned int *)pOutBuf = bufLen;
rzt = propsSize + destLen + 4;
}else sRzt.resize(0);

}
    catch (std::exception &e) {
        printf("lzma_pack_buf err [%s]:: buf len %d, rzt[%d] \n", e.what(), bufLen, rzt);
    }
if( fDebug ) printf("lzma_pack_buf:: res [%d], propsSize[%d], destLen[%d],  rzt = [%d]\n", res, propsSize, destLen, rzt);
return rzt;
}

int lzma_pack_block(CBlock* block, string& sRzt, int iLevel, unsigned int iDictSize)  // (1 << 17) = 131072 = 128K
{
int rzt = 0;
string sbf;
int bsz = CBlockToBuffer(block, sbf);
if( bsz > 12 )
{
unsigned char* pBuf = (unsigned char*)sbf.c_str();
rzt = lzma_pack_buf(pBuf, bsz, sRzt, iLevel, iDictSize);
//if( lzRzt > 0 ){ rzt = lzRzt; }  // + 4; }
}
sbf.resize(0);
return rzt;
}

int bitnet_pack_block(CBlock* block, string& sRzt)
{
if( dw_zip_block == 1 )  return lzma_pack_block(block, sRzt, 9, uint_256KB);
else if( dw_zip_block == 2 ) return lz4_pack_block(block, sRzt);
}

bool getCBlockByFilePos(CAutoFile filein, unsigned int nBlockPos, CBlock* block)
{
bool rzt = false;
int ips = nBlockPos - 4;  // get ziped block size;
if (fseek(filein, ips, SEEK_SET) != 0)
return error("getCBlockByFilePos:: fseek failed");
filein >> ips; // get ziped block size;
if( fDebug )printf("getCBlockByFilePos:: ziped block size [%d] \n", ips);
string s;   s.resize(ips);   char* pZipBuf = (char *)s.c_str();
filein.read(pZipBuf, ips);
string sUnpak;
int iRealSz;
if( dw_zip_block == 1 ) iRealSz = lzma_depack_buf((unsigned char*)pZipBuf, ips, sUnpak);
else if( dw_zip_block == 2 ) iRealSz = lz4_unpack_buf(pZipBuf, ips - 4, sUnpak);
if( fDebug )printf("getCBlockByFilePos:: ziped block size [%d], iRealSz [%d] \n", ips, iRealSz);
if( iRealSz > 0 )
{
pZipBuf = (char *)sUnpak.c_str();
rzt = CBlockFromBuffer(block, pZipBuf, iRealSz) > 12;
/*if( fDebug ){
if( block->vtx.size() < 10 )
{
printf("\n\n getCBlockByFilePos:: block info (%d): \n", rzt);
block->print();
}else printf("\n\n getCBlockByFilePos:: block vtx count (%d) is too large \n", block->vtx.size());
}*/
}
s.resize(0);   sUnpak.resize(0);
return rzt;
}

bool getCBlocksTxByFilePos(CAutoFile filein, unsigned int nBlockPos, unsigned int txId, CTransaction& tx)
{
bool rzt = false;
CBlock block;
rzt = getCBlockByFilePos(filein, nBlockPos, &block);
if( rzt )
{
if( block.vtx.size() > txId )
{
tx = block.vtx[txId];
if( fDebug ){
printf("\n\n getCBlocksTxByFilePos:: tx info: \n");
tx.print(); }
}else rzt = false;
}
return rzt;
}

int dw_zip_block = 0;
int dw_zip_limit_size = 0;
int dw_zip_txdb = 0;

bool AppInit2()
{
...
    // ********************************************************* Step 2: parameter interactions

#ifdef WIN32
    dw_zip_block = GetArg("-zipblock", 1);
#else
    /* 7Zip source code in the Linux system needs to improve, It can work, but sometimes it will crash. */
    dw_zip_block = GetArg("-zipblock", 0);
#endif
    dw_zip_limit_size = GetArg("-ziplimitsize", 64);
    dw_zip_txdb = GetArg("-ziptxdb", 0);
    if( dw_zip_block > 1 ){ dw_zip_block = 1; }
    else if( dw_zip_block == 0 ){ dw_zip_txdb = 0; }

...
}
 

extern int bitnet_pack_block(CBlock* block, string& sRzt);
extern bool getCBlockByFilePos(CAutoFile filein, unsigned int nBlockPos, CBlock* block);
extern bool getCBlocksTxByFilePos(CAutoFile filein, unsigned int nBlockPos, unsigned int txId, CTransaction& tx);
extern int dw_zip_block;

class CTransaction
{
...
    bool ReadFromDisk(CDiskTxPos pos, FILE** pfileRet=NULL)
    {
        CAutoFile filein = CAutoFile(OpenBlockFile(pos.nFile, 0, pfileRet ? "rb+" : "rb"), SER_DISK, CLIENT_VERSION);
        if (!filein)
            return error("CTransaction::ReadFromDisk() : OpenBlockFile failed");

        if( dw_zip_block > 0 )
{
//if( fDebug ) printf("CTransaction::ReadFromDisk():: pos.nFile [%d], nBlockPos [%d], nTxPos [%d], pfileRet [%d] \n", pos.nFile, pos.nBlockPos, pos.nTxPos, pfileRet);
getCBlocksTxByFilePos(filein, pos.nBlockPos, pos.nTxPos, *this);
}else{
        // Read transaction
        if (fseek(filein, pos.nTxPos, SEEK_SET) != 0)
            return error("CTransaction::ReadFromDisk() : fseek failed");

        try {
            filein >> *this;
        }
        catch (std::exception &e) {
            return error("%s() : deserialize or I/O error", __PRETTY_FUNCTION__);
        }}

        // Return file pointer
        if (pfileRet)
        {
            if (fseek(filein, pos.nTxPos, SEEK_SET) != 0)
                return error("CTransaction::ReadFromDisk() : second fseek failed");
            *pfileRet = filein.release();
        }
        return true;
    }
...
}

class CBlock
{
...
    bool WriteToDisk(unsigned int& nFileRet, unsigned int& nBlockPosRet, bool bForceWrite = false)
    {
        // Open history file to append
        CAutoFile fileout = CAutoFile(AppendBlockFile(nFileRet), SER_DISK, CLIENT_VERSION);
        if (!fileout)
            return error("CBlock::WriteToDisk() : AppendBlockFile failed");

        // Write index header
        unsigned int nSize = fileout.GetSerializeSize(*this);

        int nSize2 = nSize;
string sRzt;
        if( dw_zip_block > 0 )
        {
// compression blcok +++
nSize = bitnet_pack_block(this, sRzt);  // nSize include 4 byte( block Real size )
// compression blcok +++
        }

        fileout << FLATDATA(pchMessageStart) << nSize;

        // Write block
        long fileOutPos = ftell(fileout);
        if (fileOutPos < 0)
            return error("CBlock::WriteToDisk() : ftell failed");
        nBlockPosRet = fileOutPos;

        if( dw_zip_block == 0 ){ fileout << *this; }
        else{
   //if( fDebug ) printf("main.h Block.WriteToDisk:: nFileRet [%d], nBlockSize [%d], zipBlockSize [%d], nBlockPosRet = [%d] \n", nFileRet, nSize2, nSize, nBlockPosRet);
// compression blcok +++
   if( nSize > 0 ){
fileout.write(sRzt.c_str(), nSize);
}
sRzt.resize(0);
// compression blcok +++
        }

        // Flush stdio buffers and commit to disk before returning
        fflush(fileout);
        if( bForceWrite || (!IsInitialBlockDownload() || (nBestHeight+1) % 500 == 0) )
            FileCommit(fileout);

        return true;
    }

    bool ReadFromDisk(unsigned int nFile, unsigned int nBlockPos, bool fReadTransactions=true)
    {
        SetNull();
unsigned int iPos = nBlockPos;
if( dw_zip_block > 0 ){ iPos = 0; }

        // Open history file to read
        CAutoFile filein = CAutoFile(OpenBlockFile(nFile, iPos, "rb"), SER_DISK, CLIENT_VERSION);
        if (!filein)
            return error("CBlock::ReadFromDisk() : OpenBlockFile failed");
        if (!fReadTransactions)
            filein.nType |= SER_BLOCKHEADERONLY;

        // Read block
        try {
            if( dw_zip_block > 0 )
            {
getCBlockByFilePos(filein, nBlockPos, this);
}else{ filein >> *this; }
        }
        catch (std::exception &e) {
            return error("%s() : deserialize or I/O error", __PRETTY_FUNCTION__);
        }

        // Check the header
        if (fReadTransactions && IsProofOfWork() && !CheckProofOfWork(GetPoWHash(), nBits))
            return error("CBlock::ReadFromDisk() : errors in block header");

        return true;
    }

...
}