Topic: the fastest possible way to mass-generate addresses in Python - page 2. (Read 1377 times)

$ g++ gen.cpp
gen.cpp: In function ‘int main()’:
gen.cpp:11:23: warning: ISO C++ forbids converting a string constant to ‘char*’ [-Wwrite-strings]
   11 |     privKey.SetBase10("1");
      |                       ^~~
/usr/bin/ld: /tmp/ccVxtbUk.o: in function `main':
gen.cpp:(.text+0x32): undefined reference to `Secp256K1::Secp256K1()'
/usr/bin/ld: gen.cpp:(.text+0x48): undefined reference to `Secp256K1::Init()'
/usr/bin/ld: gen.cpp:(.text+0x57): undefined reference to `Int::Int()'
/usr/bin/ld: gen.cpp:(.text+0x70): undefined reference to `Int::SetBase10(char*)'
/usr/bin/ld: gen.cpp:(.text+0x7f): undefined reference to `Point::Point()'
/usr/bin/ld: gen.cpp:(.text+0xea): undefined reference to `Secp256K1::ComputePublicKey(Int*)'
/usr/bin/ld: gen.cpp:(.text+0x1cb): undefined reference to `Point::~Point()'
/usr/bin/ld: gen.cpp:(.text+0x1f5): undefined reference to `Secp256K1::GetAddress[abi:cxx11](int, bool, Point&)'
/usr/bin/ld: gen.cpp:(.text+0x252): undefined reference to `Int::AddOne()'
/usr/bin/ld: gen.cpp:(.text+0x2aa): undefined reference to `Point::~Point()'
/usr/bin/ld: gen.cpp:(.text+0x319): undefined reference to `Point::~Point()'
collect2: error: ld returned 1 exit status

$ gcc -o gen_cpp gen.cpp 

$ gcc -o test test.cpp 
test.cpp: In function ‘int main()’:
test.cpp:11:23: warning: ISO C++ forbids converting a string constant to ‘char*’ [-Wwrite-strings]
   11 |     privKey.SetBase10("1");
      |                       ^~~
/usr/bin/ld: /tmp/ccU4lCiy.o: in function `main':
test.cpp:(.text+0x27): undefined reference to `operator new(unsigned long)'
/usr/bin/ld: test.cpp:(.text+0x32): undefined reference to `Secp256K1::Secp256K1()'
/usr/bin/ld: test.cpp:(.text+0x48): undefined reference to `Secp256K1::Init()'
/usr/bin/ld: test.cpp:(.text+0x57): undefined reference to `Int::Int()'
/usr/bin/ld: test.cpp:(.text+0x70): undefined reference to `Int::SetBase10(char*)'
/usr/bin/ld: test.cpp:(.text+0x7f): undefined reference to `Point::Point()'
/usr/bin/ld: test.cpp:(.text+0x8e): undefined reference to `std::__cxx11::basic_string, std::allocator >::basic_string()'
/usr/bin/ld: test.cpp:(.text+0x9d): undefined reference to `std::basic_ofstream >::basic_ofstream()'
/usr/bin/ld: test.cpp:(.text+0xbb): undefined reference to `std::basic_ofstream >::open(char const*, std::_Ios_Openmode)'
/usr/bin/ld: test.cpp:(.text+0xea): undefined reference to `Secp256K1::ComputePublicKey(Int*)'
/usr/bin/ld: test.cpp:(.text+0x1cb): undefined reference to `Point::~Point()'
/usr/bin/ld: test.cpp:(.text+0x1f5): undefined reference to `Secp256K1::GetAddress[abi:cxx11](int, bool, Point&)'
/usr/bin/ld: test.cpp:(.text+0x20e): undefined reference to `std::__cxx11::basic_string, std::allocator >::operator=(std::__cxx11::basic_string, std::allocator >&&)'
/usr/bin/ld: test.cpp:(.text+0x21d): undefined reference to `std::__cxx11::basic_string, std::allocator >::~basic_string()'
/usr/bin/ld: test.cpp:(.text+0x236): undefined reference to `std::basic_ostream >& std::operator<< , std::allocator >(std::basic_ostream >&, std::__cxx11::basic_string, std::allocator > const&)'
/usr/bin/ld: test.cpp:(.text+0x243): undefined reference to `std::basic_ostream >& std::operator<<  >(std::basic_ostream >&, char)'
/usr/bin/ld: test.cpp:(.text+0x252): undefined reference to `Int::AddOne()'
/usr/bin/ld: test.cpp:(.text+0x278): undefined reference to `std::basic_ofstream >::close()'
/usr/bin/ld: test.cpp:(.text+0x28c): undefined reference to `std::basic_ofstream >::~basic_ofstream()'
/usr/bin/ld: test.cpp:(.text+0x29b): undefined reference to `std::__cxx11::basic_string, std::allocator >::~basic_string()'
/usr/bin/ld: test.cpp:(.text+0x2aa): undefined reference to `Point::~Point()'
/usr/bin/ld: test.cpp:(.text+0x2d1): undefined reference to `operator delete(void*, unsigned long)'
/usr/bin/ld: test.cpp:(.text+0x2f2): undefined reference to `std::basic_ofstream >::~basic_ofstream()'
/usr/bin/ld: test.cpp:(.text+0x30a): undefined reference to `std::__cxx11::basic_string, std::allocator >::~basic_string()'
/usr/bin/ld: test.cpp:(.text+0x319): undefined reference to `Point::~Point()'
/usr/bin/ld: /tmp/ccU4lCiy.o: in function `__static_initialization_and_destruction_0(int, int)':
test.cpp:(.text+0x365): undefined reference to `std::ios_base::Init::Init()'
/usr/bin/ld: test.cpp:(.text+0x380): undefined reference to `std::ios_base::Init::~Init()'
/usr/bin/ld: /tmp/ccU4lCiy.o:(.data.rel.local.DW.ref.__gxx_personality_v0[DW.ref.__gxx_personality_v0]+0x0): undefined reference to `__gxx_personality_v0'
collect2: error: ld returned 1 exit status

$ gcc -o test test.cpp 

test.cpp: In function ‘int main()’:
test.cpp:18:30: error: ‘class Secp256K1’ has no member named ‘GetAddress’
   18 |         bitAddr = secp256k1->GetAddress(0, false, pub);
      |                              ^~~~~~~~~~

$ ls -l

drwx------ secp256k1
-rw-rw---- test.cpp

$ ls -l secp256k1/

-rw------- Int.cpp
-rw------- IntGroup.cpp
-rw------- IntGroup.h
-rw------- Int.h
-rw------- IntMod.cpp
-rw------- Point.cpp
-rw------- Point.h
-rw------- Random.cpp
-rw------- Random.h
-rw------- SECP256K1.cpp
-rw------- SECP256k1.h

secrets.randbelow(2**256)

#!/usr/bin/env python3
# 2023/Jan/01, citb0in_multicore_secrets.py
import concurrent.futures
import os
import numpy as np
import secrets
import secp256k1 as ice

# how many cores to use
#num_cores = 1
num_cores = os.cpu_count()

# Set the number of addresses to generate
num_addresses = 1000000

# Define a worker function that generates a batch of addresses and returns them
def worker(start, end):
  # Generate a NumPy array of random private keys using "secrets" library
  private_keys = np.array([secrets.randbelow(2**256) for _ in range(start, end)])

  # Use secp256k1 to convert the private keys to addresses
  thread_addresses = np.array([ice.privatekey_to_address(2, True, dec) for dec in private_keys])

  return thread_addresses

# Use a ProcessPoolExecutor to generate the addresses in parallel
with concurrent.futures.ProcessPoolExecutor() as executor:
  # Divide the addresses evenly among the available CPU cores
  addresses_per_core = num_addresses // num_cores

  # Submit a task for each batch of addresses to the executor
  tasks = []
  for i in range(num_cores):
    start = i * addresses_per_core
    end = (i+1) * addresses_per_core
    tasks.append(executor.submit(worker, start, end))

  # Wait for the tasks to complete and retrieve the results
  addresses = []
  for task in concurrent.futures.as_completed(tasks):
    addresses.extend(task.result())

# Write the addresses to a file
np.savetxt('addresses_1M_multicore_secrets.txt', addresses, fmt='%s')

Private key : 0000000000000000000000000000000000000000000000000000000000000001
Public key  : 
x: 79be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798 
y: 483ada7726a3c4655da4fbfc0e1108a8fd17b448a68554199c47d08ffb10d4b8
 

PrKey WIF c.: KwDiBf89QgGbjEhKnhXJuH7LrciVrZi3qYjgd9M7rFU73sVHnoWn
Address before b58 encode  : 751e76e8199196d454941c45d1b3a323f1433bd6
Address   with b58 encode  : 1BgGZ9tcN4rm9KBzDn7KprQz87SZ26SAMH

1 core, 1M of addresses
time python3 gen_batches.py  > addresses.out

real	0m3,961s
user	0m3,910s
sys	0m0,055s


16 cores, 1M of addresses
time python3 gen_batches.py  > addresses.out

real	0m0,527s
user	0m7,404s
sys	0m0,151s



1 core, 16.4M of addresses
time python3 gen_batches.py  > addresses.out

real	1m1,757s
user	1m1,382s
sys	0m0,348s


16 cores, 16.4M of addresses
time python3 gen_batches.py  > addresses.out

real	0m6,672s
user	2m2,989s
sys	0m0,698s


less addresses.out

c71c6741ae4862dadaf2d9c9295d47410a27e194
1d9a3ef1efeec75b05f0ece73ad9402cac767843
44abfeb8701c716380ed2a40aeb72370ef61d7aa
d6a2aa6799f42078faa889dcda92b29c6005d84d
f3a0e804269feeba68d8c1facf68206b73b2a987
7a795a7b4500f80101e489e9ad5d7bc1855edf13
d8e619b13be4c2d6182aa10c0e7237a35ba0ab05
be801790f2cefb432bdf5dce2946fd22364b36b0
531a9c231f3e42d4bdb02d44bf01104e6eccd83d
7c8391a816b578f0d6e0a56595b67e12a314f945
6d0b359ba6f3d382eb99a346f99f0d27c29e2963
aa4116886a8631699f5077d78f47e6013a1af05f
2574970ffa9d5b26d52d89a76eef9c3c1bfbe798
...

#!/usr/bin/env python3
# 2022/Dec/26, [b]citb0in_multithreaded.py[/b]
import threading
import numpy as np
import fastecdsa.keys as fkeys
import fastecdsa.curve as fcurve
import secp256k1 as ice

# Set the number of addresses to generate and the number of threads to use
num_addresses = 1000000
num_threads = 16

# Divide the addresses evenly among the threads
addresses_per_thread = num_addresses // num_threads

# Create a list to store the generated addresses
addresses = []

# Define a worker function that generates a batch of addresses and stores them in the shared list
def worker(start, end):
  # Generate a NumPy array of random private keys using fastecdsa
  private_keys = np.array([fkeys.gen_private_key(fcurve.P256) for _ in range(start, end)])

  # Use secp256k1 to convert the private keys to addresses
  thread_addresses = np.array([ice.privatekey_to_address(2, True, dec) for dec in private_keys])

  # Add the addresses to the shared list
  addresses.extend(thread_addresses)

# Create a list of threads
threads = []

# Create and start the threads
for i in range(num_threads):
  start = i * addresses_per_thread
  end = (i+1) * addresses_per_thread
  t = threading.Thread(target=worker, args=(start, end))
  threads.append(t)
  t.start()

# Wait for the threads to finish
for t in threads:
  t.join()

# Write the addresses to a file
np.savetxt('addresses_1M_multithreaded.txt', addresses, fmt='%s')

$ time python3 citb0in_multithreaded.py 

#!/usr/bin/env python3
# 2022/Dec/30, [b]citb0in_singlethreaded.py[/b]
[...]
num_threads = 1
[...]

$ time python3 citb0in_singlethreaded.py 

#!/usr/bin/env python3
# 2022/Dec/30, [b]citb0in_multicore.py[/b]
import concurrent.futures
import os
import numpy as np
import fastecdsa.keys as fkeys
import fastecdsa.curve as fcurve
import secp256k1 as ice

# how many cores to use
#num_cores = 1
num_cores = os.cpu_count()

# Set the number of addresses to generate
num_addresses = 1000000

# Define a worker function that generates a batch of addresses and returns them
def worker(start, end):
  # Generate a NumPy array of random private keys using fastecdsa
  private_keys = np.array([fkeys.gen_private_key(fcurve.P256) for _ in range(start, end)])

  # Use secp256k1 to convert the private keys to addresses
  thread_addresses = np.array([ice.privatekey_to_address(2, True, dec) for dec in private_keys])

  return thread_addresses

# Use a ProcessPoolExecutor to generate the addresses in parallel
with concurrent.futures.ProcessPoolExecutor() as executor:
  # Divide the addresses evenly among the available CPU cores
  addresses_per_core = num_addresses // num_cores

  # Submit a task for each batch of addresses to the executor
  tasks = []
  for i in range(num_cores):
    start = i * addresses_per_core
    end = (i+1) * addresses_per_core
    tasks.append(executor.submit(worker, start, end))

  # Wait for the tasks to complete and retrieve the results
  addresses = []
  for task in concurrent.futures.as_completed(tasks):
    addresses.extend(task.result())

# Write the addresses to a file
np.savetxt('addresses_1M_multicore.txt', addresses, fmt='%s')

#!/usr/bin/env python3
# 2022/Dec/30, [b]citb0in_singlecore.py[/b]
[...]
num_cores = 1
#num_cores = os.cpu_count()
[...]
np.savetxt('addresses_1M_singlecore.txt', addresses, fmt='%s')

#!/usr/bin/env python3
# 2022/Dec/26, [b]citb0in_seq.py[/b]
import concurrent.futures
import os
import numpy as np
import secp256k1 as ice

# how many cores to use
num_cores = 1
#num_cores = os.cpu_count()

# Number of addresses to generate
num_addresses = 1000000

# Starting point (decimal/integer) for private key
starting_point = 123456789

# Define a worker function that generates a batch of addresses and returns them
def worker(start, end, starting_point):
  # Initialize the private key to the starting point
  private_key = starting_point

  # Initialize the list to hold to private keys
  private_keys = []

  # Generate a batch of private keys sequentially
  for i in range(start, end):
    # Increment the private key
    private_key += 1

    # Add the private key to the list
    private_keys.append(private_key)

  # Use secp256k1 to convert the private keys to addresses
  thread_addresses = np.array([ice.privatekey_to_address(2, True, dec) for dec in private_keys])

  return thread_addresses
  #return (thread_addresses, private_keys, start_int)

# Use a ProcessPoolExecutor to generate the addresses in parallel
with concurrent.futures.ProcessPoolExecutor() as executor:
  # Divide the addresses evenly among the available CPU cores
  addresses_per_core = num_addresses // num_cores

  # Submit a task for each batch of addresses to the executor
  tasks = []
  for i in range(num_cores):
    start = i * addresses_per_core
    end = (i+1) * addresses_per_core
    tasks.append(executor.submit(worker, start, end, starting_point))

  # Wait for the tasks to complete and retrieve the results
  addresses = []
  for task in concurrent.futures.as_completed(tasks):
    addresses.extend(task.result())

# Write the addresses to a file
np.savetxt('addresses_1M_seq_singlecore.txt', addresses, fmt='%s')

[...]
#num_cores = 1
num_cores = os.cpu_count()
[...]
# Write the addresses to a file
np.savetxt('addresses_1M_seq_singlecore.txt', addresses, fmt='%s')