Fix bugs and imports

bitcoin/compute_metrics.py

@@ -14,6 +14,7 @@
 from network_decentralization.metrics.nakamoto_coefficient import compute_nakamoto_coefficient
 from network_decentralization.metrics.entropy import compute_entropy
 from network_decentralization.metrics.concentration_ratio import compute_concentration_ratio
+from network_decentralization.metrics.total_entities import compute_total_entities
 
 
 def read_csv_data(csv_path):

cardano/compute_metrics.py

@@ -13,6 +13,7 @@
 from metrics.nakamoto_coefficient import compute_nakamoto_coefficient
 from metrics.entropy import compute_entropy
 from metrics.concentration_ratio import compute_concentration_ratio
+from metrics.total_entities import compute_total_entities
 
 
 def read_csv_data(csv_path):

cardano/metrics/entropy.py

@@ -1,5 +1,5 @@
 from math import log
-from network_decentralization.metrics.total_entities import compute_total_entities
+from metrics.total_entities import compute_total_entities
 
 
 def compute_entropy(distribution, alpha):

cardano/metrics/nakamoto_coefficient.py

@@ -1,4 +1,4 @@
-from network_decentralization.metrics.tau_index import compute_tau_index
+from metrics.tau_index import compute_tau_index
 
 
 def compute_nakamoto_coefficient(distribution):

cardano/parse.py

@@ -9,7 +9,7 @@
 
 Nodes are handled as follows:
 - Onion addresses (.onion): categorised as 'Tor'
-- Unresolved DNS names: categorised as 'Unresolved' (addresses that couldn't be resolved to IPs)
+- Unresolved DNS names: categorised as 'Unknown' (addresses that couldn't be resolved to IPs)
 - Missing geodata: categorised as 'Unknown' (IPs with no geolocation data available)
 - Geodata errors: categorised as 'Unknown' (error entries in the geodata file)
 - Valid entries: categorised by their country, organization, or ASN
@@ -45,7 +45,8 @@ def load_cardano_nodes():
 
     with open(relays_file, 'r') as f:
         pool_relays = json.load(f)
-    for pool_id, relays in pool_relays.items():
+
+    for _, relays in pool_relays.items():
         for relay in relays:
             ip = relay.get('ipv4') or relay.get('ipv6')
             port = relay.get('port', 3001)
@@ -57,7 +58,7 @@ def load_cardano_nodes():
                 if resolved_ip:
                     nodes.append((resolved_ip, port))
                 else:
-                    # Always include unresolved DNS as (dns_name, port)
+                    # Keep unresolved DNS names so they can later be categorised as Unknown.
                     nodes.append((dns_name, port))
     return nodes
 
@@ -101,28 +102,20 @@ def get_geodata(mode='Countries'):
     ledger = 'cardano'
     output_dir = hlp.get_output_directory()
     geodata_file = output_dir / 'geodata' / f'{ledger}.json'
-    nodes_file = output_dir / 'cardano_extracted_nodes.json'
-
-    # Load unresolved count
-    unresolved_count = 0
-    if nodes_file.exists():
-        with open(nodes_file) as f:
-            data = json.load(f)
-            unresolved_count = data.get('unresolved_count', 0)
-
+
     if not geodata_file.exists():
         logging.error(f'Geodata file not found: {geodata_file}')
         logging.info('Please run collect_cardano_geodata.py first!')
-        return {}, unresolved_count
-    
+        return {}
+
     with open(geodata_file) as f:
         geodata = json.load(f)
-    
+
     # Deduplicate nodes by (ip/dns, port)
     nodes = list(set(load_cardano_nodes()))
     categories = defaultdict(list)
 
-    # Load unresolved DNS names from dns_resolved.json
+    # Load unresolved DNS names
     dns_resolved_file = Path(__file__).parent / 'output' / 'dns_resolved.json'
     unresolved_dns = set()
     if dns_resolved_file.exists():
@@ -131,142 +124,137 @@ def get_geodata(mode='Countries'):
                 if entry.get('ip_address') == 'Unresolved':
                     unresolved_dns.add((entry.get('dns_name'), entry.get('port', 3001)))
 
-    for node in nodes:
-        ip_addr = node[0]
-        port = node[1] if len(node) > 1 else 3001
+    for ip_addr, port in nodes:
 
-        # If this node is an unresolved DNS, count as 'Unresolved'
+        # DNS names that could not be resolved are treated as Unknown
         if (ip_addr, port) in unresolved_dns:
-            categories['Unresolved'].append(ip_addr)
+            categories['Unknown'].append(ip_addr)
             continue
 
-        if ip_addr in geodata:
-            ip_info = geodata[ip_addr]
-
-            # Skip error entries
-            if 'error' in ip_info and ip_info['error']:
-                categories['Unknown'].append(ip_addr)
-                continue
-
-            if mode == 'Countries':
-                try:
-                    country = ip_info.get('country') or ip_info.get('location', {}).get('country')
-                    if country:
-                        categories[country].append(ip_addr)
-                    else:
-                        categories['Unknown'].append(ip_addr)
-                except (KeyError, TypeError):
-                    categories['Unknown'].append(ip_addr)
-
-            elif mode == 'ASN':
-                try:
-                    # Extract ASN number (AS14061) instead of organization name
-                    asn = None
-                    if 'as' in ip_info and ip_info['as']:
-                        # Format: "AS14061 DigitalOcean, LLC"
-                        asn_parts = ip_info['as'].split(None, 1)  # Split on first whitespace
-                        if asn_parts:
-                            asn = asn_parts[0]  # Get the AS number
-                    elif 'asn' in ip_info and ip_info['asn']:
-                        asn = "AS" + str(ip_info['asn'].get('asn', ''))
-
-                    if asn:
-                        categories[asn].append(ip_addr)
-                    else:
-                        categories['Unknown'].append(ip_addr)
-                except (KeyError, TypeError, AttributeError, IndexError):
-                    categories['Unknown'].append(ip_addr)
-
-            elif mode == 'Organizations':
-                try:
-                    # Try org field first, then extract from AS field
-                    org_name = ip_info.get('org') or ip_info.get('asn', {}).get('org')
-
-                    # If no org field, try to extract from AS field
-                    if not org_name and 'as' in ip_info and ip_info['as']:
-                        asn_parts = ip_info['as'].split(None, 1)  # Split on first whitespace
-                        if len(asn_parts) > 1:
-                            org_name = asn_parts[1]  # Get everything after ASN number
-
-                    if org_name:
-                        clustered_name = cluster_org_name(org_name)
-                        categories[clustered_name].append(ip_addr)
-                    else:
-                        categories['Unknown'].append(ip_addr)
-                except (KeyError, TypeError):
-                    categories['Unknown'].append(ip_addr)
-
-        elif ip_addr.endswith('onion'):
+        if ip_addr.endswith('.onion'):
             categories['Tor'].append(ip_addr)
-        else:
+            continue
+
+        if ip_addr not in geodata:
             categories['Unknown'].append(ip_addr)
+            continue
+
+        ip_info = geodata[ip_addr]
+
+        # Error entries
+        if ip_info.get('error'):
+            categories['Unknown'].append(ip_addr)
+            continue
+
+        if mode == 'Countries':
+            country = (
+                ip_info.get('country')
+                or ip_info.get('location', {}).get('country')
+            )
+
+            categories[country if country else 'Unknown'].append(ip_addr)
 
-    return categories, len(categories['Unresolved'])
+        elif mode == 'ASN':
+            asn = None
+
+            if ip_info.get('as'):
+                asn = ip_info['as'].split(None, 1)[0]
+
+            elif ip_info.get('asn'):
+                asn_num = ip_info['asn'].get('asn')
+                if asn_num:
+                    asn = f'AS{asn_num}'
+
+            categories[asn if asn else 'Unknown'].append(ip_addr)
+
+        elif mode == 'Organizations':
+            org_name = (
+                ip_info.get('org')
+                or ip_info.get('asn', {}).get('org')
+            )
+
+            if not org_name and ip_info.get('as'):
+                parts = ip_info['as'].split(None, 1)
+                if len(parts) > 1:
+                    org_name = parts[1]
+
+            if org_name:
+                categories[cluster_org_name(org_name)].append(ip_addr)
+            else:
+                categories['Unknown'].append(ip_addr)
+
+    return categories
 
 
 def parse_geography(mode='Countries'):
     """Parse geography data and save to CSV."""
     ledger = 'cardano'
     logging.info(f'Parsing {ledger} {mode}')
-    
-    geodata, unresolved_count = get_geodata(mode)
-    
+
+    geodata = get_geodata(mode)
+
     if not geodata:
         logging.error('No geodata available!')
         return
-    
-    total_nodes = sum([len(val) for val in geodata.values()])
+
+    total_nodes = sum(len(val) for val in geodata.values())
     logging.info(f'{ledger} - Total nodes: {total_nodes}')
-    if unresolved_count > 0:
-        logging.info(f'{ledger} - Unresolved entries: {unresolved_count}')
-
+
+    unknown_count = len(geodata.get('Unknown', []))
+    if unknown_count > 0:
+        logging.info(f'{ledger} - Unknown entries: {unknown_count}')
+
     # Count by category
     geodata_counter = {}
     for key, val in sorted(geodata.items(), key=lambda x: len(x[1]), reverse=True):
         if key:
             geodata_counter[key] = len(val)
         else:
-            geodata_counter["Unknown"] = geodata_counter.get("Unknown", 0) + len(val)
-    
+            geodata_counter['Unknown'] = geodata_counter.get('Unknown', 0) + len(val)
+
     output_dir = hlp.get_output_directory()
     output_dir.mkdir(parents=True, exist_ok=True)
-    
+
     filename = output_dir / f'{mode.lower()}_{ledger}.csv'
-    
+
     # Create or update CSV with timestamp
     if filename.is_file():
         df = pd.read_csv(filename)
         geodata_csv = df[mode].tolist()
         geodata_in_order = [0] * len(geodata_csv)
-        
-        for category in geodata_counter.keys():
+
+        for category, count in geodata_counter.items():
             if category in geodata_csv:
-                geodata_in_order[geodata_csv.index(category)] = geodata_counter[category]
+                geodata_in_order[geodata_csv.index(category)] = count
             else:
                 rows, columns = df.shape
-                df.loc[rows] = [category] + [0]*(columns-1)
-                geodata_in_order.append(geodata_counter[category])
-        
+                df.loc[rows] = [category] + [0] * (columns - 1)
+                geodata_in_order.append(count)
+
         df[datetime.today().strftime('%Y-%m-%d')] = geodata_in_order
-        
+
         # Sort by the latest date column in descending order
         latest_date_col = df.columns[-1]
         df = df.sort_values(by=latest_date_col, ascending=False)
         df.to_csv(filename, index=False)
     else:
         geodata_df = pd.DataFrame.from_dict(
-            geodata_counter, 
-            orient='index', 
+            geodata_counter,
+            orient='index',
             columns=[datetime.today().strftime('%Y-%m-%d')]
         )
         geodata_df.to_csv(filename, index_label=mode)
-    
+
     logging.info(f'Saved to {filename}')
-    
+
     # Print top 10
     logging.info(f'\nTop 10 {mode}:')
-    for idx, (key, count) in enumerate(sorted(geodata_counter.items(), key=lambda x: x[1], reverse=True)[:10], 1):
-        logging.info(f'  {idx}. {key}: {count} nodes ({100*count/total_nodes:.1f}%)')
+    for idx, (key, count) in enumerate(
+        sorted(geodata_counter.items(), key=lambda x: x[1], reverse=True)[:10], 1
+    ):
+        logging.info(
+            f'  {idx}. {key}: {count} nodes ({100 * count / total_nodes:.1f}%)'
+        )
 
 
 def main():

ethereum/compute_metrics.py

@@ -13,6 +13,7 @@
 from metrics.nakamoto_coefficient import compute_nakamoto_coefficient
 from metrics.entropy import compute_entropy
 from metrics.concentration_ratio import compute_concentration_ratio
+from metrics.total_entities import compute_total_entities
 
 
 def read_csv_data(csv_path):