-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathdcrln.py
104 lines (90 loc) · 3.48 KB
/
dcrln.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
import requests
import pandas as pd
from datetime import date
import os
import json
from urllib.request import urlopen
import numpy as np
import stream
pd.set_option('display.max_columns', None)
def convert_ln_json_to_df(jsonResponse):
graph_json = (jsonResponse)
df_nodes = pd.json_normalize(graph_json['nodes'])
df_channels = pd.json_normalize(graph_json['edges'])
df_channels.channel_id = df_channels.channel_id.astype(np.int64)
df_channels.capacity = df_channels.capacity.astype(int)
for index, row in df_nodes.iterrows():
ipk = row['pub_key']
node1 = df_channels[df_channels['node1_pub']==ipk]
outCount = node1.channel_id.count().astype(int)
outCapacity = node1.capacity.sum().astype(int)
node2 = df_channels[df_channels['node2_pub']==ipk]
inCount = node2.channel_id.count().astype(int)
inCapacity = node2.capacity.sum().astype(int)
df_nodes.loc[index, 'outCount'] = outCount
df_nodes.loc[index, 'outCapacity'] = outCapacity
df_nodes.loc[index, 'inCount'] = inCount
df_nodes.loc[index, 'inCapacity'] = inCapacity
df_nodes.loc[index, 'chCount'] = inCount+outCount
return df_nodes, df_channels
url = 'https://ln-map.jholdstock.uk/api/graph'
# get api response
response = requests.get(url)
strResposne = response.text
# get today's date for file path
todayStr = str(date.today())
yearMonthStr = date.today().strftime("%Y/%m/")
# path for data
pathStr = './data/lnd-raw/' + yearMonthStr
if not os.path.exists(pathStr):
os.makedirs(pathStr)
filename = pathStr + todayStr + '.txt'
# save raw json response to a csv
with open(filename, "w", encoding="utf-8") as f:
f.write(strResposne)
url = 'https://raw.githubusercontent.com/bochinchero/dcrsnapcsv/main/data/dcrlndgraph/' + yearMonthStr + todayStr +".json"
# get api response
response = requests.get(url)
strResposne = response.text
response = urlopen(url)
data_json = json.loads(response.read())
df_nodes, df_channels = convert_ln_json_to_df(data_json)
df_nodes = df_nodes.sort_values(by='chCount', ascending=False)
pathStr = './data/lnd-nodes/' + yearMonthStr
if not os.path.exists(pathStr):
os.makedirs(pathStr)
filename = pathStr + todayStr + '.csv'
# save to csv
df_nodes.to_csv(filename)
# append total node count to the nodes stream
# check if the file exists:
streamPath = './data/stream/'
streamFile = 'countLNNodes.csv'
# check if file path exists:
if not os.path.isfile((streamPath+streamFile)):
# if it doesn't exist, create header and file
stream.appendtoCSV(['date','countLNNodes'],streamPath,streamFile)
# create data list
data = [todayStr,df_nodes['pub_key'].count()]
# update stream file
stream.appendtoCSV(data,streamPath,streamFile)
# update stream for LN channels
streamFile = 'countLNChannels.csv'
# check if file path exists:
if not os.path.isfile((streamPath+streamFile)):
# if it doesn't exist, create header and file
stream.appendtoCSV(['date','countLNChannels'],streamPath,streamFile)
# create data list
data = [todayStr,df_channels['node1_pub'].count()]
# update stream file
stream.appendtoCSV(data,streamPath,streamFile)
# update stream for LN channels
streamFile = 'sumLNCapacity.csv'
# check if file path exists:
if not os.path.isfile((streamPath+streamFile)):
# if it doesn't exist, create header and file
stream.appendtoCSV(['date','sumLNCapacity'],streamPath,streamFile)
# create data list
data = [todayStr,df_channels['capacity'].sum()]
# update stream file
stream.appendtoCSV(data,streamPath,streamFile)