Topic: Performance of Account structures in bitcoind (Read 761 times)

Account Weaknesses
Since the accounts feature was introduced, several services have used it to keep track of customer's bitcoin balances and have had the following problems:

Wallet backups are an issue; if you rely on a good backup of wallet.dat then a backup must be done every time an address is associated with an account and every time the 'move' command is used.
The accounts code does not scale up to thousands of accounts with tens of thousands of transactions, because by-account (and by-account-by-time) indices are not implemented. So many operations (like computing an account balance) require accessing every wallet transaction.
Most applications already have a customer database, implemented with MySQL or some other relational database technology. It is awkward at best to keep the bitcoin-maintained Berkely DB wallet database and the application database backed up and synchronized at all times.

    process = subprocess.Popen([r'C:\Program Files (x86)\Bitcoin\daemon\bitcoind.exe', '-testnet', '-rpcuser=test', '-rpcpassword=test1'])
    time.sleep(20) #give bitcoind time to start up; a smarter way would be to check the network connection in a loop, but for our purposes this is fine
   
    try:   
          #run various tests in a loop 
    finally:
        process.terminate()

    process = subprocess.Popen([r'C:\Program Files (x86)\Bitcoin\daemon\bitcoind.exe', '-testnet', '-rpcuser=test', '-rpcpassword=test1'])
    time.sleep(20) #give bitcoind time to start up; a smarter way would be to check the network connection in a loop, but for our purposes this is fine
   
    try:   
          #run various tests in a loop 
    finally:
        if client is not None:
            client.stop()
            time.sleep(5) 

        process.terminate()

Warning: Warning: error reading wallet.dat! All keys read correctly, but transaction data or address book entries might be missing or incorrect.

>>> client.getbalance()
Traceback (most recent call last):
  File "", line 1, in
  File "C:\Python27\lib\site-packages\jsonrpc\proxy.py", line 45, in __call__
    raise JSONRPCException(resp['error'])
JSONRPCException

import subprocess
import time
import datetime
import os
import shutil
import timeit
from jsonrpc import ServiceProxy
from random import randrange


#config

    #location of an "empty" wallet file (only the default ('') account exists)
blankWalletFile = r'C:\Users\User\AppData\Roaming\Bitcoin\testnet3\wallet.empty.dat'

    #location of the live wallet file; this file will be backed up and then restored after the test is complete
liveWalletFile = r'C:\Users\User\AppData\Roaming\Bitcoin\testnet3\wallet.dat'

    #the number of accounts to be created
account_count = 50000

    #the number of random transfers to perform between accounts
transfer_count = 10000


def resetWallet(account_count = 1):
    #back up live wallet
    shutil.copy(liveWalletFile, liveWalletFile + '.bak')

    if account_count > 1: #see if we already have a wallet file with this number of accounts
        source_wallet_file = liveWalletFile + '.' + str(account_count)
   
    if not os.path.isfile(source_wallet_file):
            print('wallet file does not exist; starting with a blank file')
            source_wallet_file = blankWalletFile
    else:
            print('wallet file exists; re-using existing wallet file')
           
    #overwrite live wallet with empty wallet
    shutil.copy(source_wallet_file, liveWalletFile)

def restoreLiveWallet():
    #make a copy of this test wallet file for future use
    shutil.copy(liveWalletFile, liveWalletFile + '.' + str(account_count))
    #overwrite test wallet with live wallet backup
    shutil.move(liveWalletFile + '.bak', liveWalletFile)

def createAccounts(client, count):
    for i in range(0, count):
        client.move('', 'account%d' % i, 1e-8)
        #print(client.getbalance('account%d' % i))

def performRandomTransfers(client, count):
    account1 = ''
    account2 = getRandomAccount(account_count)

    for i in range(0, count):
        client.move(account1, account2, 1e-8)
        account1 = account2
        account2 = getRandomAccount(account_count, account1)

def getRandomAccount(account_count, except_account = ''):
    account = except_account
   
    while account == except_account:
        account = 'account%d' % randrange(0, account_count)

    return account

def main():
    print(datetime.datetime.now().time())

    resetWallet(account_count)

    print('starting bitcoind...')
    process = subprocess.Popen([r'C:\Program Files (x86)\Bitcoin\daemon\bitcoind.exe', '-testnet', '-rpcuser=test', '-rpcpassword=test1'])
    time.sleep(10) #give bitcoind time to start up; a smarter way would be to check the network connection in a loop, but for our purposes this is fine
    print('bitcoind started')
   
    try:   
        client = ServiceProxy("http://test:test1@localhost:18332")

        actual_account_count = len(client.listaccounts())
        print ('there are currently {} accounts'.format(actual_account_count))

        to_create_count =  account_count - actual_account_count + 1

        if actual_account_count < account_count:
            print('creating %d accounts' % to_create_count)
            total_time = timeit.timeit(lambda: createAccounts(client, to_create_count), number=1)
            print('time elapsed creating {0:d} accounts: {1:.2f} seconds'.format(to_create_count, total_time))

        print(datetime.datetime.now().time())

        print 'performing %d random transfers' % transfer_count
        total_time = timeit.timeit(lambda: performRandomTransfers(client, transfer_count), number=1)
        print 'time elapsed performing {0:d} transfers: {1:.2f} seconds'.format(transfer_count, total_time)

        print('there are currently {} accounts'.format(len(client.listaccounts())))
        #print(client.listaccounts())

        print(datetime.datetime.now().time())

        print('shutting down bitcoind...')
        client.stop()
        time.sleep(5)       
           
    finally:
        process.terminate()
        restoreLiveWallet()


if __name__ == "__main__":
    main()

import subprocess
import time
import datetime
import os
import shutil
import timeit
from jsonrpc import ServiceProxy
from random import randrange


#config
transfer_count = 50  #the number of random transfers to perform between accounts
account_count = 50000 #number of existing accounts in the wallet


def performRandomTransfer(client, actual_count):
    account1 = getRandomAccount()
    account2 = getRandomAccount(account1)
       
    address = client.getaccountaddress(account2)

    if (client.getbalance(account1) > 1e-4):
        tx = client.sendfrom(account1, address, 1e-4)             
    elif client.getbalance() > 1e-4:
        tx = client.sendtoaddress(address, 1e-4)
    else:
        raise Exception('No balance available in any account') 
   
    actual_count[0] += 1

    print(tx)

def getRandomAccount(except_account = ''):
    account = except_account
   
    while account == except_account:
        account = 'account%d' % randrange(0, account_count)

    return account

def main():
    print(datetime.datetime.now().time())

    print('starting bitcoind...')
    process = subprocess.Popen([r'C:\Program Files (x86)\Bitcoin\daemon\bitcoind.exe', '-testnet', '-rpcuser=test', '-rpcpassword=test1'])
    time.sleep(20) #give bitcoind time to start up; a smarter way would be to check the network connection in a loop, but for our purposes this is fine
    print('bitcoind started')
   
    try:   
        client = ServiceProxy("http://test:test1@localhost:18332")

        print(datetime.datetime.now().time())

        print 'performing %d random transfers' % transfer_count

        actual_count = [0]
        total_time = timeit.timeit(lambda: performRandomTransfer(client, actual_count), number=transfer_count)

        print 'time elapsed performing {0:d} transfers: {1:.2f} seconds'.format(actual_count[0], total_time)
        print(datetime.datetime.now().time())
                 
    finally:
        print('shutting down bitcoind...')
        if client is not None:
            client.stop()
            time.sleep(5) 

        process.terminate()


if __name__ == "__main__":
    main()

for N = 1 to 5e4
client.move('', accountN, smallAmount)

client.move(account1, account2, smallAmount)

   account1 = getRandomAccount()
    account2 = getRandomAccount(except: account1)
        
    address = client.getaccountaddress(account2)

    if (client.getbalance(account1) > 1e-4):
        tx = client.sendfrom(account1, address, 1e-4)              
    elif client.getbalance() > 1e-4:
        tx = client.sendtoaddress(address, 1e-4)
    else:
        raise Exception('No balance available in any account')