Clean up EIA-NEMS processing

nems_database_processing/README.md

@@ -8,7 +8,7 @@ All the scripts are run sequentially from `run.sh`
 `run.sh` runs 5 python scripts that process NEMS fleet data sequentially:
 * `a_inheritance.py` 
 * `b_aeo_cleaning.py`: This script cleans raw AEO-NEMS and EIA860M files and appends planned and missing existing EIA860M units into AEO-NEMS, and also updates unit retirement years according to specified version of EIA860M
-* `c_geospatial_mapping.py`: This script maps the lon/lats of units database established in step b to their counties and FIPS. For any units that are missing lon/lats, please look up their lon/lats and manually add these units with their lon/lats to in `/Inputs/user_adjusted_units_missing_lon_lats.csv`. This step will incur errors until all units are mapped to their counties and FIPS. Any units that need manually adjusted locations should be done in this step.
+* `c_geospatial_mapping.py`: This script maps the lon/lats of units database established in step b to their counties and FIPS. For any units that are missing lon/lats, please look up their lon/lats and manually add these units with their lon/lats to in `/inputs/user_adjusted_units_missing_lon_lats.csv`. This step will incur errors until all units are mapped to their counties and FIPS. Any units that need manually adjusted locations should be done in this step.
 * `d_hydro_classification.py` 
 * `e_additional_inputs.py`, which includes `e1_set_retire_years.py`, `e2_fix_upgrades.py`, and `e3_merge_psh_dbs.py`: This script handles updated retirement years that are outdated in AEO-NEMS and EIA860M, fix upgrades, and handles other additional adjustments. Any manually adjusted retirement years should be included in `e1_set_retire_years.py`.
 
@@ -29,10 +29,13 @@ All the input files to run all 5 python scripts are now specified upfront in run
 
 
 # Other input files that probably are updated less frequently
-All located in `Inputs` folder:
+All located in `inputs` folder:
 * `county_to_reeds_region.csv`
 * `tech_to_cooling_tech_map.csv`
 
 # Output file:
-Located in `Outputs` folder. This is the final file that will be used to run ReEDS:
+Located in `outputs` folder. This is the final file that will be used to run ReEDS:
 * `ReEDS_generator_database_final_EIA-NEMS.csv`
+
+# Note:
+Sometimes when running `c_geospatial_mapping.py`, geopandas operation results in invalid geometry - point (inf, inf) or polygon (inf, inf). If this occurs, run `conda install -c conda-forge proj-data` to pre-download projection data in the current environment before rerunning the script.

nems_database_processing/a_inheritance.py

@@ -46,6 +46,7 @@
 import os
 import sys
 
+print("Starting a_inheritance.py")
 #%%----------------------------------------------------------------------------
 # Step 0: User Input and Data Specification:
 #------------------------------------------------------------------------------
@@ -76,7 +77,7 @@
 # SCRIPT STEP 1: Import the old dataframe:
 #------------------------------------------------------------------------------
 
-dfold = pd.read_csv(os.path.join('Inputs','Inheritance',gdboldname), low_memory=False)
+dfold = pd.read_csv(os.path.join('inputs','Inheritance',gdboldname), low_memory=False)
 dfold.rename(columns=COL_RENAME, inplace=True)
 
 #---this dataframe collects all the PLANT_ID/UNIT_ID combinations that are
@@ -97,7 +98,7 @@
 # SCRIPT STEP 3: Import the new dataframe:
 #------------------------------------------------------------------------------
 
-dfnew = pd.read_excel(os.path.join('Inputs','AEO_NEMS',gdbnewname))
+dfnew = pd.read_excel(os.path.join('inputs','AEO_NEMS',gdbnewname))
 
 #%%----------------------------------------------------------------------------
 # SCRIPT STEP 4a: Apply the mapper to the new dataframe:
@@ -169,5 +170,7 @@
 #%%----------------------------------------------------------------------------
 # SCRIPT STEP 5: Export the new dataframe:
 #------------------------------------------------------------------------------
-os.makedirs('Outputs', exist_ok=True)
-dfout_3.to_csv(os.path.join('Outputs', gdboutname),index=False)
+os.makedirs('outputs', exist_ok=True)
+dfout_3.to_csv(os.path.join('outputs', gdboutname),index=False)
+
+print("Finished a_inheritance.py")

nems_database_processing/b_aeo_cleaning.py

@@ -34,6 +34,8 @@ def params():
     return (dir, nems_ver, battery_duration, eia860M_ver_mon, eia860M_ver_year, append_operating_units, gdbinputname, gdboutname)
 
 def main():
+    print("Starting b_aeo_cleaning.py")
+
     (dir, nems_ver, battery_duration, eia860M_ver_mon, eia860M_ver_year, append_operating_units, gdbinputname, gdboutname) = params()
 
     # Add EIA860M planned units, missing operating units, and updated retirement years to NEMS dataset:
@@ -45,7 +47,7 @@ def main():
 
     # =========================================================================
     # Save output file:
-    nems_cleaned.to_csv(os.path.join(dir,'Outputs', gdboutname), index=False)
+    nems_cleaned.to_csv(os.path.join(dir,'outputs', gdboutname), index=False)
     # =========================================================================
 
 ################################### MAIN FUNCTION ###################################
@@ -71,7 +73,7 @@ def processAEOandEIA860(dir, nems_ver, battery_duration, eia860M_ver_mon, eia860
     # Merge current NEMS and operating EIA860M:
     nems_eia860_operating = mergeAEOandEIA860M(aeo_data, eia860M_data_operating,battery_duration,status='Operating')
     # Save temp output file:
-    nems_eia860_operating.to_csv(os.path.join(dir,'Outputs', 'merged_nems_eia860_operating.csv'), index=False)
+    nems_eia860_operating.to_csv(os.path.join(dir,'outputs', 'merged_nems_eia860_operating.csv'), index=False)
     # =========================================================================
 
     # =========================================================================
@@ -82,7 +84,7 @@ def processAEOandEIA860(dir, nems_ver, battery_duration, eia860M_ver_mon, eia860
     nems_eia860_operating_retired = mergeAEOandEIA860M(nems_eia860_operating, eia860M_data_retired, battery_duration, status='Retired')
     nems_eia860_operating_retired = nems_eia860_operating_retired[nems_eia860_operating_retired['nems']==1]
     # Save temp output file:
-    nems_eia860_operating_retired.to_csv(os.path.join(dir,'Outputs', 'merged_nems_eia860_operating_retired.csv'), index=False)
+    nems_eia860_operating_retired.to_csv(os.path.join(dir,'outputs', 'merged_nems_eia860_operating_retired.csv'), index=False)
     # =========================================================================
 
     # =========================================================================
@@ -92,7 +94,7 @@ def processAEOandEIA860(dir, nems_ver, battery_duration, eia860M_ver_mon, eia860
     # Merge current operating and retired NEMS and planned EIA860M:
     nems_eia860_operating_retired_planned = mergeAEOandEIA860M(nems_eia860_operating_retired, eia860M_planned, battery_duration, status='Planned')
     # Save temp output file:
-    nems_eia860_operating_retired_planned.to_csv(os.path.join(dir,'Outputs', 'merged_nems_eia860_operating_retired_planned.csv'), index=False)
+    nems_eia860_operating_retired_planned.to_csv(os.path.join(dir,'outputs', 'merged_nems_eia860_operating_retired_planned.csv'), index=False)
     # =========================================================================
 
     # =========================================================================
@@ -108,7 +110,7 @@ def processAEOandEIA860(dir, nems_ver, battery_duration, eia860M_ver_mon, eia860
 #####################################################################################  
 
 def cleanAEOData(dir, gdbinputname):
-    aeo_data = pd.read_csv(os.path.join(dir,'Outputs',gdbinputname))
+    aeo_data = pd.read_csv(os.path.join(dir,'outputs',gdbinputname),low_memory=False)
     aeo_data = aeo_data.astype({'T_PID':'string','T_UID':'string', 'T_SYR': 'int', 'T_RYR': 'int'})
     aeo_data['T_PID'] = aeo_data['T_PID'].str.replace(" ", "")
     aeo_data['T_UID'] = aeo_data['T_UID'].str.replace(" ", "")
@@ -119,7 +121,7 @@ def cleanAEOData(dir, gdbinputname):
     return aeo_data
 
 def cleanEIA860MData(dir, ver_mon, ver_year, battery_duration, status):
-    eia860M_data = pd.read_excel(os.path.join(dir,'Inputs','EIA860M',ver_mon+'_generator'+str(ver_year)+'.xlsx'), 
+    eia860M_data = pd.read_excel(os.path.join(dir,'inputs','EIA860M',ver_mon+'_generator'+str(ver_year)+'.xlsx'), 
                                  sheet_name=status, header=1, index_col=False)
     if ver_year >=2020:
         eia860M_data.columns = eia860M_data.iloc[0]
@@ -214,7 +216,7 @@ def cleanEIA860MData(dir, ver_mon, ver_year, battery_duration, status):
     eia860M_data.loc[eia860M_data['Technology'].str.contains("Landfill", na=False),'tech'] = 'lfill-gas'
 
     # Add wst to match with NEMS:
-    cooling_tech = pd.read_csv(os.path.join(dir,'Inputs','tech_to_cooling_tech_map.csv'))
+    cooling_tech = pd.read_csv(os.path.join(dir,'inputs','tech_to_cooling_tech_map.csv'))
     eia860M_data = pd.merge(eia860M_data, cooling_tech, on=['tech'], how='left')
 
     # Clean up:
@@ -329,16 +331,16 @@ def cleanMergedAEOEIA860(merged_nems_eia860, battery_duration):
     merged_nems_eia860['TC_WIN'] = merged_nems_eia860['TC_WIN'] * merged_nems_eia860['TCOUNT']
     merged_nems_eia860['TC_SUM'] = merged_nems_eia860['TC_SUM'] * merged_nems_eia860['TCOUNT']
 
-    merged_nems_eia860['TC_NP'] = merged_nems_eia860['TC_NP'].round(2)
-    merged_nems_eia860['TC_WIN'] = merged_nems_eia860['TC_WIN'].round(2)
-    merged_nems_eia860['TC_SUM'] = merged_nems_eia860['TC_SUM'].round(2)
+    rounding_cols = ['TC_NP', 'TC_WIN','TC_SUM','T_VOM','T_FOM',
+                     'T_CCSROV','T_CCSF','T_CCSV','T_CCSHR','T_CAPAD']
+    merged_nems_eia860[rounding_cols] = merged_nems_eia860[rounding_cols].round(2)
 
     ## Further clean up
     # Add heat rate for EIA860M units:
     nems_eia860_final = addHeatrates(merged_nems_eia860)
 
     # Assign tech to missing tech values:
-    techmap = pd.read_excel(os.path.join('Inputs','NEMS to ReEDS Tech Mapping.xlsx'))
+    techmap = pd.read_excel(os.path.join('inputs','NEMS to ReEDS Tech Mapping.xlsx'))
     techmap.rename(columns={'EFD Code':'EFDcd', 'ReEDS Tech':'tech'}, inplace=True)
     nems_eia860_final = nems_eia860_final.merge(techmap, on=['EFDcd'], how='left')
     nems_eia860_final['tech'] = np.where(((nems_eia860_final['tech_x'].isnull()) | (nems_eia860_final['tech_x']=='others')),
@@ -347,7 +349,7 @@ def cleanMergedAEOEIA860(merged_nems_eia860, battery_duration):
 
     # coal-new technologies are scrubbed coal units with an online data of 1995 of later
     coal_new_filter = (nems_eia860_final['tech'].isin(['coaloldscr'])) & (nems_eia860_final['TRFURB'] >= 1995)
-    nems_eia860_final['tech'][coal_new_filter] = 'coal-new'
+    nems_eia860_final.loc[coal_new_filter, 'tech'] = 'coal-new'
 
     # Assign energy capacity to batteries that are not in EIA860M but are in NEMS
     nems_eia860_final.loc[((nems_eia860_final['tech'].str.contains('battery')) | 
@@ -375,3 +377,5 @@ def cleanMergedAEOEIA860(merged_nems_eia860, battery_duration):
     return nems_eia860_final
 
 main()
+
+print("Finished b_aeo_cleaning.py")