From e7f6c388eb759aaf51383426e01e355bd4420284 Mon Sep 17 00:00:00 2001 From: LauraAntunes1 <190111637+LauraAntunes1@users.noreply.github.com> Date: Mon, 13 Jul 2026 12:22:29 +0100 Subject: [PATCH 1/2] Modify/Add imports for metrics files --- bitcoin/compute_metrics.py | 1 + cardano/compute_metrics.py | 1 + cardano/metrics/entropy.py | 2 +- cardano/metrics/nakamoto_coefficient.py | 2 +- ethereum/compute_metrics.py | 1 + 5 files changed, 5 insertions(+), 2 deletions(-) diff --git a/bitcoin/compute_metrics.py b/bitcoin/compute_metrics.py index 8c76f74..ded22ab 100644 --- a/bitcoin/compute_metrics.py +++ b/bitcoin/compute_metrics.py @@ -14,6 +14,7 @@ from network_decentralization.metrics.nakamoto_coefficient import compute_nakamoto_coefficient from network_decentralization.metrics.entropy import compute_entropy from network_decentralization.metrics.concentration_ratio import compute_concentration_ratio +from network_decentralization.metrics.total_entities import compute_total_entities def read_csv_data(csv_path): diff --git a/cardano/compute_metrics.py b/cardano/compute_metrics.py index 630b6ce..231b12d 100644 --- a/cardano/compute_metrics.py +++ b/cardano/compute_metrics.py @@ -13,6 +13,7 @@ from metrics.nakamoto_coefficient import compute_nakamoto_coefficient from metrics.entropy import compute_entropy from metrics.concentration_ratio import compute_concentration_ratio +from metrics.total_entities import compute_total_entities def read_csv_data(csv_path): diff --git a/cardano/metrics/entropy.py b/cardano/metrics/entropy.py index 96e2c38..d6aac1e 100644 --- a/cardano/metrics/entropy.py +++ b/cardano/metrics/entropy.py @@ -1,5 +1,5 @@ from math import log -from network_decentralization.metrics.total_entities import compute_total_entities +from metrics.total_entities import compute_total_entities def compute_entropy(distribution, alpha): diff --git a/cardano/metrics/nakamoto_coefficient.py b/cardano/metrics/nakamoto_coefficient.py index e0bd938..a369383 100644 --- a/cardano/metrics/nakamoto_coefficient.py +++ b/cardano/metrics/nakamoto_coefficient.py @@ -1,4 +1,4 @@ -from network_decentralization.metrics.tau_index import compute_tau_index +from metrics.tau_index import compute_tau_index def compute_nakamoto_coefficient(distribution): diff --git a/ethereum/compute_metrics.py b/ethereum/compute_metrics.py index b5823d8..39ccfcf 100644 --- a/ethereum/compute_metrics.py +++ b/ethereum/compute_metrics.py @@ -13,6 +13,7 @@ from metrics.nakamoto_coefficient import compute_nakamoto_coefficient from metrics.entropy import compute_entropy from metrics.concentration_ratio import compute_concentration_ratio +from metrics.total_entities import compute_total_entities def read_csv_data(csv_path): From 2fea51407e61c663587649e02e3632e88b4ed688 Mon Sep 17 00:00:00 2001 From: LauraAntunes1 <190111637+LauraAntunes1@users.noreply.github.com> Date: Mon, 13 Jul 2026 13:15:13 +0100 Subject: [PATCH 2/2] Removing Unresolved category for Cardano nodes --- cardano/parse.py | 198 ++++++++++++++++++++++------------------------- 1 file changed, 93 insertions(+), 105 deletions(-) diff --git a/cardano/parse.py b/cardano/parse.py index d019d13..a85a08d 100644 --- a/cardano/parse.py +++ b/cardano/parse.py @@ -9,7 +9,7 @@ Nodes are handled as follows: - Onion addresses (.onion): categorised as 'Tor' -- Unresolved DNS names: categorised as 'Unresolved' (addresses that couldn't be resolved to IPs) +- Unresolved DNS names: categorised as 'Unknown' (addresses that couldn't be resolved to IPs) - Missing geodata: categorised as 'Unknown' (IPs with no geolocation data available) - Geodata errors: categorised as 'Unknown' (error entries in the geodata file) - Valid entries: categorised by their country, organization, or ASN @@ -45,7 +45,8 @@ def load_cardano_nodes(): with open(relays_file, 'r') as f: pool_relays = json.load(f) - for pool_id, relays in pool_relays.items(): + + for _, relays in pool_relays.items(): for relay in relays: ip = relay.get('ipv4') or relay.get('ipv6') port = relay.get('port', 3001) @@ -57,7 +58,7 @@ def load_cardano_nodes(): if resolved_ip: nodes.append((resolved_ip, port)) else: - # Always include unresolved DNS as (dns_name, port) + # Keep unresolved DNS names so they can later be categorised as Unknown. nodes.append((dns_name, port)) return nodes @@ -101,28 +102,20 @@ def get_geodata(mode='Countries'): ledger = 'cardano' output_dir = hlp.get_output_directory() geodata_file = output_dir / 'geodata' / f'{ledger}.json' - nodes_file = output_dir / 'cardano_extracted_nodes.json' - - # Load unresolved count - unresolved_count = 0 - if nodes_file.exists(): - with open(nodes_file) as f: - data = json.load(f) - unresolved_count = data.get('unresolved_count', 0) - + if not geodata_file.exists(): logging.error(f'Geodata file not found: {geodata_file}') logging.info('Please run collect_cardano_geodata.py first!') - return {}, unresolved_count - + return {} + with open(geodata_file) as f: geodata = json.load(f) - + # Deduplicate nodes by (ip/dns, port) nodes = list(set(load_cardano_nodes())) categories = defaultdict(list) - # Load unresolved DNS names from dns_resolved.json + # Load unresolved DNS names dns_resolved_file = Path(__file__).parent / 'output' / 'dns_resolved.json' unresolved_dns = set() if dns_resolved_file.exists(): @@ -131,142 +124,137 @@ def get_geodata(mode='Countries'): if entry.get('ip_address') == 'Unresolved': unresolved_dns.add((entry.get('dns_name'), entry.get('port', 3001))) - for node in nodes: - ip_addr = node[0] - port = node[1] if len(node) > 1 else 3001 + for ip_addr, port in nodes: - # If this node is an unresolved DNS, count as 'Unresolved' + # DNS names that could not be resolved are treated as Unknown if (ip_addr, port) in unresolved_dns: - categories['Unresolved'].append(ip_addr) + categories['Unknown'].append(ip_addr) continue - if ip_addr in geodata: - ip_info = geodata[ip_addr] - - # Skip error entries - if 'error' in ip_info and ip_info['error']: - categories['Unknown'].append(ip_addr) - continue - - if mode == 'Countries': - try: - country = ip_info.get('country') or ip_info.get('location', {}).get('country') - if country: - categories[country].append(ip_addr) - else: - categories['Unknown'].append(ip_addr) - except (KeyError, TypeError): - categories['Unknown'].append(ip_addr) - - elif mode == 'ASN': - try: - # Extract ASN number (AS14061) instead of organization name - asn = None - if 'as' in ip_info and ip_info['as']: - # Format: "AS14061 DigitalOcean, LLC" - asn_parts = ip_info['as'].split(None, 1) # Split on first whitespace - if asn_parts: - asn = asn_parts[0] # Get the AS number - elif 'asn' in ip_info and ip_info['asn']: - asn = "AS" + str(ip_info['asn'].get('asn', '')) - - if asn: - categories[asn].append(ip_addr) - else: - categories['Unknown'].append(ip_addr) - except (KeyError, TypeError, AttributeError, IndexError): - categories['Unknown'].append(ip_addr) - - elif mode == 'Organizations': - try: - # Try org field first, then extract from AS field - org_name = ip_info.get('org') or ip_info.get('asn', {}).get('org') - - # If no org field, try to extract from AS field - if not org_name and 'as' in ip_info and ip_info['as']: - asn_parts = ip_info['as'].split(None, 1) # Split on first whitespace - if len(asn_parts) > 1: - org_name = asn_parts[1] # Get everything after ASN number - - if org_name: - clustered_name = cluster_org_name(org_name) - categories[clustered_name].append(ip_addr) - else: - categories['Unknown'].append(ip_addr) - except (KeyError, TypeError): - categories['Unknown'].append(ip_addr) - - elif ip_addr.endswith('onion'): + if ip_addr.endswith('.onion'): categories['Tor'].append(ip_addr) - else: + continue + + if ip_addr not in geodata: categories['Unknown'].append(ip_addr) + continue + + ip_info = geodata[ip_addr] + + # Error entries + if ip_info.get('error'): + categories['Unknown'].append(ip_addr) + continue + + if mode == 'Countries': + country = ( + ip_info.get('country') + or ip_info.get('location', {}).get('country') + ) + + categories[country if country else 'Unknown'].append(ip_addr) - return categories, len(categories['Unresolved']) + elif mode == 'ASN': + asn = None + + if ip_info.get('as'): + asn = ip_info['as'].split(None, 1)[0] + + elif ip_info.get('asn'): + asn_num = ip_info['asn'].get('asn') + if asn_num: + asn = f'AS{asn_num}' + + categories[asn if asn else 'Unknown'].append(ip_addr) + + elif mode == 'Organizations': + org_name = ( + ip_info.get('org') + or ip_info.get('asn', {}).get('org') + ) + + if not org_name and ip_info.get('as'): + parts = ip_info['as'].split(None, 1) + if len(parts) > 1: + org_name = parts[1] + + if org_name: + categories[cluster_org_name(org_name)].append(ip_addr) + else: + categories['Unknown'].append(ip_addr) + + return categories def parse_geography(mode='Countries'): """Parse geography data and save to CSV.""" ledger = 'cardano' logging.info(f'Parsing {ledger} {mode}') - - geodata, unresolved_count = get_geodata(mode) - + + geodata = get_geodata(mode) + if not geodata: logging.error('No geodata available!') return - - total_nodes = sum([len(val) for val in geodata.values()]) + + total_nodes = sum(len(val) for val in geodata.values()) logging.info(f'{ledger} - Total nodes: {total_nodes}') - if unresolved_count > 0: - logging.info(f'{ledger} - Unresolved entries: {unresolved_count}') - + + unknown_count = len(geodata.get('Unknown', [])) + if unknown_count > 0: + logging.info(f'{ledger} - Unknown entries: {unknown_count}') + # Count by category geodata_counter = {} for key, val in sorted(geodata.items(), key=lambda x: len(x[1]), reverse=True): if key: geodata_counter[key] = len(val) else: - geodata_counter["Unknown"] = geodata_counter.get("Unknown", 0) + len(val) - + geodata_counter['Unknown'] = geodata_counter.get('Unknown', 0) + len(val) + output_dir = hlp.get_output_directory() output_dir.mkdir(parents=True, exist_ok=True) - + filename = output_dir / f'{mode.lower()}_{ledger}.csv' - + # Create or update CSV with timestamp if filename.is_file(): df = pd.read_csv(filename) geodata_csv = df[mode].tolist() geodata_in_order = [0] * len(geodata_csv) - - for category in geodata_counter.keys(): + + for category, count in geodata_counter.items(): if category in geodata_csv: - geodata_in_order[geodata_csv.index(category)] = geodata_counter[category] + geodata_in_order[geodata_csv.index(category)] = count else: rows, columns = df.shape - df.loc[rows] = [category] + [0]*(columns-1) - geodata_in_order.append(geodata_counter[category]) - + df.loc[rows] = [category] + [0] * (columns - 1) + geodata_in_order.append(count) + df[datetime.today().strftime('%Y-%m-%d')] = geodata_in_order - + # Sort by the latest date column in descending order latest_date_col = df.columns[-1] df = df.sort_values(by=latest_date_col, ascending=False) df.to_csv(filename, index=False) else: geodata_df = pd.DataFrame.from_dict( - geodata_counter, - orient='index', + geodata_counter, + orient='index', columns=[datetime.today().strftime('%Y-%m-%d')] ) geodata_df.to_csv(filename, index_label=mode) - + logging.info(f'Saved to {filename}') - + # Print top 10 logging.info(f'\nTop 10 {mode}:') - for idx, (key, count) in enumerate(sorted(geodata_counter.items(), key=lambda x: x[1], reverse=True)[:10], 1): - logging.info(f' {idx}. {key}: {count} nodes ({100*count/total_nodes:.1f}%)') + for idx, (key, count) in enumerate( + sorted(geodata_counter.items(), key=lambda x: x[1], reverse=True)[:10], 1 + ): + logging.info( + f' {idx}. {key}: {count} nodes ({100 * count / total_nodes:.1f}%)' + ) def main():