maker.py

import psycopg2
import json
import pandas as pd
from web3 import Web3, HTTPProvider
from datetime import datetime
from multiprocessing import Process

#url of Ethereum node
urlETH = ""
#connect to Ethereum client 
w3 = Web3(HTTPProvider(urlETH))

try:
    connection = psycopg2.connect(user="",
                                  password="",
                                  host="",
                                  port="",
                                  database="eth_db")
    cursor = connection.cursor()
    getTokenDelegates ="""copy (SELECT SUBSTRING(topic1,13,20) AS delegate, SUBSTRING(topic2,13,20) AS voteDelegate From logs Where address ='\\xD897F108670903D1d6070fcf818f9db3615AF272' AND topic0 ='\\x2187b96b95fffefab01016c852705bc8ec76d1ea17dd5bffef25fd7136633644' ) to stdout"""
    with open("data/delegates/mkr.csv", 'w') as fp:
        cursor.copy_expert(getTokenDelegates, fp)
            
except (Exception, psycopg2.Error) as error:
    print(error)
    
finally:
    if connection:
        cursor.close()
        connection.close()
        
lastFiveSpells=[
    "0xcD672aCc9885796a19b4bAf03Dba46c8cdB0882B",
    "0xdD0AB42848b5609ded083459110d2e38483E0859",
    "0xD8D60b7A9998098261DF5175B5b0Fb567CD0Fb1A", 
    "0xdB2C426173e5a9c10af3CD834B87DEAad40525Ff",
    "0x0E9AB92e3Fad77eE35a5f702Ac56c48bAAB7B0eE"
]

def getDataFromNodeMKR(all,w3,directory,spell): 
    abi = json.load(open('abi/MKRSpell.abi'))
    contract = w3.eth.contract(address=w3.to_checksum_address(spell), abi=abi)
    abi = json.load(open('abi/MKRChief.abi'))
    chief = w3.eth.contract(address=w3.to_checksum_address('0x0a3f6849f78076aefaDf113F5BED87720274dDC0'), abi=abi)
    eta = contract.functions.eta().call()
    distribution = {}
    try:
        connection = psycopg2.connect(user="hlioba",password="TeRK8B5kg9@#29",host="localhost",port="5432",database="eth_db")
        cursor = connection.cursor()
        ts =eta
        cursor.execute("Select max(height) from blocks where time_mined<='{}'".format(datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')))
        blockNumber =cursor.fetchall()[0][0]
 
    except (Exception, psycopg2.Error) as error:
        print(error)
        
    finally:
        if connection:
            cursor.close()
            connection.close()
            
    for address in all:
        data = chief.functions.deposits(w3.to_checksum_address(address)).call({}, block_identifier=blockNumber)
        distribution[address] = data
        
    tokenDistributionDF = pd.DataFrame(distribution.items(), columns=['holder', 'votes'])
    tokenDistributionDF['votes'] = tokenDistributionDF['votes'].apply(lambda x: x/(10**18))
    tokenDistributionDF=tokenDistributionDF[tokenDistributionDF['votes']>0]
    tokenDistributionDF.to_pickle(directory+spell+'.pkl')

delegates =pd.read_csv("data/delegates/mkr.csv",sep="\t",names=["delegate","voteDelegate"])
delegates['voteDelegate'] = delegates['voteDelegate'].apply(lambda x: '0'+ x[2:])
delegates = delegates["voteDelegate"].tolist()
holders= pd.read_csv("data/holders/mkr.csv",delimiter ="\t",names=['holder'])
holders = holders['holder'].apply(lambda x: '0'+ x[2:]).values.tolist()
all = list(set(delegates+holders))
directory='data/spells/'
for i in lastFiveSpells:
    processes = []
for spell in lastFiveSpells:
    p = Process(target=getDataFromNodeMKR, args=(all,w3,directory,spell), )
    processes.append(p)
    # start process
    p.start()
    # join processes
for process in processes:
    process.join()
    processes.remove(process)