Topic: the fastest possible way to mass-generate addresses in Python (Read 1208 times)

    with open("addresses_1M_multicore_secrets" + str(i) + ".txt", "a") as f:
      f.write(f'{thred_addresses}n')

private_keys = list(map(secrets.randbelow,n))

#!/usr/bin/env python3
# 2023/Jan/01, citb0in_multicore_secrets.py
import concurrent.futures
import os
import secrets
import secp256k1 as ice

# how many cores to use
num_cores = 10
#num_cores = os.cpu_count()

# Set the number of addresses to generate
num_addresses = 10000000


# Define a worker function that generates a batch of addresses and returns them
def worker(start, end, i):
 
  addr_type = [2] * (end - start)
  is_compressed = [True] * (end - start)
  n = [0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141] * (end - start) 
 
  f = open("addresses_1M_multicore_secrets" + str(i) + ".txt", "w")
  
  # Generate a list of random private keys using "secrets" library
  private_keys = list(map(secrets.randbelow,n))

  
  # Use secp256k1 to convert the private keys to addresses
  thread_addresses = list(map(ice.privatekey_to_address, addr_type, is_compressed, private_keys))
  
  # Write the addresses in the thread file
  list(map(lambda x:f.write(x+"\n"),thread_addresses))

  f.close()
  
  return
  

# Use a ProcessPoolExecutor to generate the addresses in parallel
with concurrent.futures.ProcessPoolExecutor() as executor:
  # Divide the addresses evenly among the available CPU cores
  addresses_per_core = num_addresses // num_cores
   
  
  # Submit a task for each batch of addresses to the executor
  tasks = []
  for i in range(num_cores):
    start = i * addresses_per_core
    end = (i+1) * addresses_per_core
    tasks.append(executor.submit(worker, start, end, i))

$ time python3 citb0in_multicore_secrets.py 

$ time python3 citb0in_multicore_secrets_splitsave.py 

# Starting point (decimal/integer) for private key
starting_point = 123456789

# Define a worker function that generates a batch of addresses and returns them
def worker(start, end, starting_point):
  # Initialize the private key to the starting point
  private_key = starting_point

  # Initialize the list to hold to private keys
  private_keys = []

  # Generate a batch of private keys sequentially
  for i in range(start, end):
    # Increment the private key
    private_key += 1 #TODO why not use a large prime number instead of 1? 1 is not random at all, but a few dozen bits varying at once can be made to look random to others.

#!/usr/bin/env python3
# 2023/Jan/01, citb0in_multicore_secrets.py
import concurrent.futures
import os
import numpy as np
import secrets
import secp256k1 as ice

# how many cores to use
#num_cores = 1
num_cores = os.cpu_count()

# Set the number of addresses to generate
num_addresses = 1000000

# Define a worker function that generates a batch of addresses and returns them
def worker(start, end, i):
  # Generate a NumPy array of random private keys using "secrets" library
  private_keys = np.array([secrets.randbelow(2**256) for _ in range(start, end)])

  # Use secp256k1 to convert the private keys to addresses
  thread_addresses = np.array([ice.privatekey_to_address(2, True, dec) for dec in private_keys])
  np.savetxt('addresses_1M_multicore_secrets'+ str(i) +'.txt', thread_addresses, fmt='%s')
  
  return 

# Use a ProcessPoolExecutor to generate the addresses in parallel
with concurrent.futures.ProcessPoolExecutor() as executor:
  # Divide the addresses evenly among the available CPU cores
  addresses_per_core = num_addresses // num_cores
  
  # Submit a task for each batch of addresses to the executor
  tasks = []
  for i in range(num_cores):
    start = i * addresses_per_core
    end = (i+1) * addresses_per_core
    tasks.append(executor.submit(worker, start, end, i))

$ g++ gen.cpp
gen.cpp: In function ‘int main()’:
gen.cpp:11:23: warning: ISO C++ forbids converting a string constant to ‘char*’ [-Wwrite-strings]
   11 |     privKey.SetBase10("1");
      |                       ^~~
/usr/bin/ld: /tmp/ccVxtbUk.o: in function `main':
gen.cpp:(.text+0x32): undefined reference to `Secp256K1::Secp256K1()'
/usr/bin/ld: gen.cpp:(.text+0x48): undefined reference to `Secp256K1::Init()'
/usr/bin/ld: gen.cpp:(.text+0x57): undefined reference to `Int::Int()'
/usr/bin/ld: gen.cpp:(.text+0x70): undefined reference to `Int::SetBase10(char*)'
/usr/bin/ld: gen.cpp:(.text+0x7f): undefined reference to `Point::Point()'
/usr/bin/ld: gen.cpp:(.text+0xea): undefined reference to `Secp256K1::ComputePublicKey(Int*)'
/usr/bin/ld: gen.cpp:(.text+0x1cb): undefined reference to `Point::~Point()'
/usr/bin/ld: gen.cpp:(.text+0x1f5): undefined reference to `Secp256K1::GetAddress[abi:cxx11](int, bool, Point&)'
/usr/bin/ld: gen.cpp:(.text+0x252): undefined reference to `Int::AddOne()'
/usr/bin/ld: gen.cpp:(.text+0x2aa): undefined reference to `Point::~Point()'
/usr/bin/ld: gen.cpp:(.text+0x319): undefined reference to `Point::~Point()'
collect2: error: ld returned 1 exit status

$ gcc -o gen_cpp gen.cpp 

$ gcc -o test test.cpp 
test.cpp: In function ‘int main()’:
test.cpp:11:23: warning: ISO C++ forbids converting a string constant to ‘char*’ [-Wwrite-strings]
   11 |     privKey.SetBase10("1");
      |                       ^~~
/usr/bin/ld: /tmp/ccU4lCiy.o: in function `main':
test.cpp:(.text+0x27): undefined reference to `operator new(unsigned long)'
/usr/bin/ld: test.cpp:(.text+0x32): undefined reference to `Secp256K1::Secp256K1()'
/usr/bin/ld: test.cpp:(.text+0x48): undefined reference to `Secp256K1::Init()'
/usr/bin/ld: test.cpp:(.text+0x57): undefined reference to `Int::Int()'
/usr/bin/ld: test.cpp:(.text+0x70): undefined reference to `Int::SetBase10(char*)'
/usr/bin/ld: test.cpp:(.text+0x7f): undefined reference to `Point::Point()'
/usr/bin/ld: test.cpp:(.text+0x8e): undefined reference to `std::__cxx11::basic_string, std::allocator >::basic_string()'
/usr/bin/ld: test.cpp:(.text+0x9d): undefined reference to `std::basic_ofstream >::basic_ofstream()'
/usr/bin/ld: test.cpp:(.text+0xbb): undefined reference to `std::basic_ofstream >::open(char const*, std::_Ios_Openmode)'
/usr/bin/ld: test.cpp:(.text+0xea): undefined reference to `Secp256K1::ComputePublicKey(Int*)'
/usr/bin/ld: test.cpp:(.text+0x1cb): undefined reference to `Point::~Point()'
/usr/bin/ld: test.cpp:(.text+0x1f5): undefined reference to `Secp256K1::GetAddress[abi:cxx11](int, bool, Point&)'
/usr/bin/ld: test.cpp:(.text+0x20e): undefined reference to `std::__cxx11::basic_string, std::allocator >::operator=(std::__cxx11::basic_string, std::allocator >&&)'
/usr/bin/ld: test.cpp:(.text+0x21d): undefined reference to `std::__cxx11::basic_string, std::allocator >::~basic_string()'
/usr/bin/ld: test.cpp:(.text+0x236): undefined reference to `std::basic_ostream >& std::operator<< , std::allocator >(std::basic_ostream >&, std::__cxx11::basic_string, std::allocator > const&)'
/usr/bin/ld: test.cpp:(.text+0x243): undefined reference to `std::basic_ostream >& std::operator<<  >(std::basic_ostream >&, char)'
/usr/bin/ld: test.cpp:(.text+0x252): undefined reference to `Int::AddOne()'
/usr/bin/ld: test.cpp:(.text+0x278): undefined reference to `std::basic_ofstream >::close()'
/usr/bin/ld: test.cpp:(.text+0x28c): undefined reference to `std::basic_ofstream >::~basic_ofstream()'
/usr/bin/ld: test.cpp:(.text+0x29b): undefined reference to `std::__cxx11::basic_string, std::allocator >::~basic_string()'
/usr/bin/ld: test.cpp:(.text+0x2aa): undefined reference to `Point::~Point()'
/usr/bin/ld: test.cpp:(.text+0x2d1): undefined reference to `operator delete(void*, unsigned long)'
/usr/bin/ld: test.cpp:(.text+0x2f2): undefined reference to `std::basic_ofstream >::~basic_ofstream()'
/usr/bin/ld: test.cpp:(.text+0x30a): undefined reference to `std::__cxx11::basic_string, std::allocator >::~basic_string()'
/usr/bin/ld: test.cpp:(.text+0x319): undefined reference to `Point::~Point()'
/usr/bin/ld: /tmp/ccU4lCiy.o: in function `__static_initialization_and_destruction_0(int, int)':
test.cpp:(.text+0x365): undefined reference to `std::ios_base::Init::Init()'
/usr/bin/ld: test.cpp:(.text+0x380): undefined reference to `std::ios_base::Init::~Init()'
/usr/bin/ld: /tmp/ccU4lCiy.o:(.data.rel.local.DW.ref.__gxx_personality_v0[DW.ref.__gxx_personality_v0]+0x0): undefined reference to `__gxx_personality_v0'
collect2: error: ld returned 1 exit status

$ gcc -o test test.cpp 

test.cpp: In function ‘int main()’:
test.cpp:18:30: error: ‘class Secp256K1’ has no member named ‘GetAddress’
   18 |         bitAddr = secp256k1->GetAddress(0, false, pub);
      |                              ^~~~~~~~~~

$ ls -l

drwx------ secp256k1
-rw-rw---- test.cpp

$ ls -l secp256k1/

-rw------- Int.cpp
-rw------- IntGroup.cpp
-rw------- IntGroup.h
-rw------- Int.h
-rw------- IntMod.cpp
-rw------- Point.cpp
-rw------- Point.h
-rw------- Random.cpp
-rw------- Random.h
-rw------- SECP256K1.cpp
-rw------- SECP256k1.h

secrets.randbelow(2**256)

#!/usr/bin/env python3
# 2023/Jan/01, citb0in_multicore_secrets.py
import concurrent.futures
import os
import numpy as np
import secrets
import secp256k1 as ice

# how many cores to use
#num_cores = 1
num_cores = os.cpu_count()

# Set the number of addresses to generate
num_addresses = 1000000

# Define a worker function that generates a batch of addresses and returns them
def worker(start, end):
  # Generate a NumPy array of random private keys using "secrets" library
  private_keys = np.array([secrets.randbelow(2**256) for _ in range(start, end)])

  # Use secp256k1 to convert the private keys to addresses
  thread_addresses = np.array([ice.privatekey_to_address(2, True, dec) for dec in private_keys])

  return thread_addresses

# Use a ProcessPoolExecutor to generate the addresses in parallel
with concurrent.futures.ProcessPoolExecutor() as executor:
  # Divide the addresses evenly among the available CPU cores
  addresses_per_core = num_addresses // num_cores

  # Submit a task for each batch of addresses to the executor
  tasks = []
  for i in range(num_cores):
    start = i * addresses_per_core
    end = (i+1) * addresses_per_core
    tasks.append(executor.submit(worker, start, end))

  # Wait for the tasks to complete and retrieve the results
  addresses = []
  for task in concurrent.futures.as_completed(tasks):
    addresses.extend(task.result())

# Write the addresses to a file
np.savetxt('addresses_1M_multicore_secrets.txt', addresses, fmt='%s')

Private key : 0000000000000000000000000000000000000000000000000000000000000001
Public key  : 
x: 79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798 
y: 483ada7726a3c4655da4fbfc0e1108a8fd17b448a68554199c47d08ffb10d4b8
 

PrKey WIF c.: KwDiBf89QgGbjEhKnhXJuH7LrciVrZi3qYjgd9M7rFU73sVHnoWn
Address before b58 encode  : 751e76e8199196d454941c45d1b3a323f1433bd6
Address   with b58 encode  : 1BgGZ9tcN4rm9KBzDn7KprQz87SZ26SAMH

1 core, 1M of addresses
time python3 gen_batches.py  > addresses.out

real	0m3,961s
user	0m3,910s
sys	0m0,055s


16 cores, 1M of addresses
time python3 gen_batches.py  > addresses.out

real	0m0,527s
user	0m7,404s
sys	0m0,151s



1 core, 16.4M of addresses
time python3 gen_batches.py  > addresses.out

real	1m1,757s
user	1m1,382s
sys	0m0,348s


16 cores, 16.4M of addresses
time python3 gen_batches.py  > addresses.out

real	0m6,672s
user	2m2,989s
sys	0m0,698s


less addresses.out

c71c6741ae4862dadaf2d9c9295d47410a27e194
1d9a3ef1efeec75b05f0ece73ad9402cac767843
44abfeb8701c716380ed2a40aeb72370ef61d7aa
d6a2aa6799f42078faa889dcda92b29c6005d84d
f3a0e804269feeba68d8c1facf68206b73b2a987
7a795a7b4500f80101e489e9ad5d7bc1855edf13
d8e619b13be4c2d6182aa10c0e7237a35ba0ab05
be801790f2cefb432bdf5dce2946fd22364b36b0
531a9c231f3e42d4bdb02d44bf01104e6eccd83d
7c8391a816b578f0d6e0a56595b67e12a314f945
6d0b359ba6f3d382eb99a346f99f0d27c29e2963
aa4116886a8631699f5077d78f47e6013a1af05f
2574970ffa9d5b26d52d89a76eef9c3c1bfbe798
...

#!/usr/bin/env python3
# 2022/Dec/26, [b]citb0in_multithreaded.py[/b]
import threading
import numpy as np
import fastecdsa.keys as fkeys
import fastecdsa.curve as fcurve
import secp256k1 as ice

# Set the number of addresses to generate and the number of threads to use
num_addresses = 1000000
num_threads = 16

# Divide the addresses evenly among the threads
addresses_per_thread = num_addresses // num_threads

# Create a list to store the generated addresses
addresses = []

# Define a worker function that generates a batch of addresses and stores them in the shared list
def worker(start, end):
  # Generate a NumPy array of random private keys using fastecdsa
  private_keys = np.array([fkeys.gen_private_key(fcurve.P256) for _ in range(start, end)])

  # Use secp256k1 to convert the private keys to addresses
  thread_addresses = np.array([ice.privatekey_to_address(2, True, dec) for dec in private_keys])

  # Add the addresses to the shared list
  addresses.extend(thread_addresses)

# Create a list of threads
threads = []

# Create and start the threads
for i in range(num_threads):
  start = i * addresses_per_thread
  end = (i+1) * addresses_per_thread
  t = threading.Thread(target=worker, args=(start, end))
  threads.append(t)
  t.start()

# Wait for the threads to finish
for t in threads:
  t.join()

# Write the addresses to a file
np.savetxt('addresses_1M_multithreaded.txt', addresses, fmt='%s')

$ time python3 citb0in_multithreaded.py 

#!/usr/bin/env python3
# 2022/Dec/30, [b]citb0in_singlethreaded.py[/b]
[...]
num_threads = 1
[...]

$ time python3 citb0in_singlethreaded.py 

#!/usr/bin/env python3
# 2022/Dec/30, [b]citb0in_multicore.py[/b]
import concurrent.futures
import os
import numpy as np
import fastecdsa.keys as fkeys
import fastecdsa.curve as fcurve
import secp256k1 as ice

# how many cores to use
#num_cores = 1
num_cores = os.cpu_count()

# Set the number of addresses to generate
num_addresses = 1000000

# Define a worker function that generates a batch of addresses and returns them
def worker(start, end):
  # Generate a NumPy array of random private keys using fastecdsa
  private_keys = np.array([fkeys.gen_private_key(fcurve.P256) for _ in range(start, end)])

  # Use secp256k1 to convert the private keys to addresses
  thread_addresses = np.array([ice.privatekey_to_address(2, True, dec) for dec in private_keys])

  return thread_addresses

# Use a ProcessPoolExecutor to generate the addresses in parallel
with concurrent.futures.ProcessPoolExecutor() as executor:
  # Divide the addresses evenly among the available CPU cores
  addresses_per_core = num_addresses // num_cores

  # Submit a task for each batch of addresses to the executor
  tasks = []
  for i in range(num_cores):
    start = i * addresses_per_core
    end = (i+1) * addresses_per_core
    tasks.append(executor.submit(worker, start, end))

  # Wait for the tasks to complete and retrieve the results
  addresses = []
  for task in concurrent.futures.as_completed(tasks):
    addresses.extend(task.result())

# Write the addresses to a file
np.savetxt('addresses_1M_multicore.txt', addresses, fmt='%s')

#!/usr/bin/env python3
# 2022/Dec/30, [b]citb0in_singlecore.py[/b]
[...]
num_cores = 1
#num_cores = os.cpu_count()
[...]
np.savetxt('addresses_1M_singlecore.txt', addresses, fmt='%s')

#!/usr/bin/env python3
# 2022/Dec/26, [b]citb0in_seq.py[/b]
import concurrent.futures
import os
import numpy as np
import secp256k1 as ice

# how many cores to use
num_cores = 1
#num_cores = os.cpu_count()

# Number of addresses to generate
num_addresses = 1000000

# Starting point (decimal/integer) for private key
starting_point = 123456789

# Define a worker function that generates a batch of addresses and returns them
def worker(start, end, starting_point):
  # Initialize the private key to the starting point
  private_key = starting_point

  # Initialize the list to hold to private keys
  private_keys = []

  # Generate a batch of private keys sequentially
  for i in range(start, end):
    # Increment the private key
    private_key += 1

    # Add the private key to the list
    private_keys.append(private_key)

  # Use secp256k1 to convert the private keys to addresses
  thread_addresses = np.array([ice.privatekey_to_address(2, True, dec) for dec in private_keys])

  return thread_addresses
  #return (thread_addresses, private_keys, start_int)

# Use a ProcessPoolExecutor to generate the addresses in parallel
with concurrent.futures.ProcessPoolExecutor() as executor:
  # Divide the addresses evenly among the available CPU cores
  addresses_per_core = num_addresses // num_cores

  # Submit a task for each batch of addresses to the executor
  tasks = []
  for i in range(num_cores):
    start = i * addresses_per_core
    end = (i+1) * addresses_per_core
    tasks.append(executor.submit(worker, start, end, starting_point))

  # Wait for the tasks to complete and retrieve the results
  addresses = []
  for task in concurrent.futures.as_completed(tasks):
    addresses.extend(task.result())

# Write the addresses to a file
np.savetxt('addresses_1M_seq_singlecore.txt', addresses, fmt='%s')

[...]
#num_cores = 1
num_cores = os.cpu_count()
[...]
# Write the addresses to a file
np.savetxt('addresses_1M_seq_singlecore.txt', addresses, fmt='%s')

time python3 gen_batches.py > addresses.out

real	0m29,810s
user	8m22,402s
sys	0m6,273s

wc addresses.out

 16474040  16474040 575934502

less addresses.out

18GW1MFwpYZgmRSy8R4wgjUtBnscJtXeUZ
1FXUP9HViWxrtgAvqbpgVrd69yarx6Njdp
1MuxV99fRFnhoNopdxV7kkQB4u14chCtqA
17AzXPYExdE8nxkiV6zJGLuJq32WkAZ134
12DUcGmFMkmXV8tYihM2jLQ36pc4zLBDP6
1AqwPisfi9hDHoF95SEoSEVN89fX8NSGjo
17qRbvdShkoCDw3XJGhEqxaFkQHaK36Yk1
187x8hPiVpRLGDRsvy4nreeMNW7aW9zuFv
1EGSDnc2j7p3qsPJsPakKgavCTz4szYU3w
1F4KjKLNqQSbmw9xsJurRVQ3RciL7Kox7g
1AKaHBfhMEedrtnjGgmFFsdG3SzyN3hpcZ
1GaWuuPPM747Na3CTwaR3DcewoBHc9VVSv
12VeWT4jZCep17HJswMtfCEqh14oDcRr3L
16MBC6WLWMvNeexYYX853Ucf2ZsPoYLRoS
1JVrhzEKTcgzGtSc9u9LpFjksFZiQveFT6
17kz2Wpry3vnLM6RMjB3KXoG6kRxjeDQcj
1TBh3q7vQoTmgX6pY75ADHDSkcaWkmCvd
13QVDFeeWkErhTAFqX3SP2YzrzWt3Bh7iu
1GPyzJC8Ey8PasHn2Qa8aFanjrXuE1aNK5
17QE8iNrtLnju5pinG1vdoCVZpn8zbrM7S
17nKRy1XdKdroWXPsnsd4RvxWKVySZxath
1KWaY8K824oEFk5q4tZioCZ86viLJuKNgS
141m3BUZNjMxbs4a9Cb6SbCmEzx4nPJKCL
12ueqsLkT1XCVheGM34uQWW2fpnhVvtwmE
14eH2SFvCq3fxmrK9NgKTxfMCeKixNwQ6c
1G9CaAqoKJSfTRyGcvPhVFewLRXKWzkwh7
15RthH7DBuDSmYcFCYanUhBsrdEYzBahNw
13BTmQ2gCaZ8x4T7Sba9k2MRBSDRrZLy6u
1NpbKVze4pPVCghjaa2CV7LfRkotCNu76Y
1HfQ42v7DEDHEuQMhH5caPzX2RgtSmoYWw
...