From 499edf5737b623f3f193364192df73d4000a15c2 Mon Sep 17 00:00:00 2001 From: cdeline Date: Tue, 14 Mar 2023 06:03:50 -0600 Subject: [PATCH 1/6] Update mismatch.py mad_fn to handle arrays and dataframes with 1D calculations. --- bifacial_radiance/mismatch.py | 19 +++++++++++++------ tests/test_mismatch.py | 11 +++++++---- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/bifacial_radiance/mismatch.py b/bifacial_radiance/mismatch.py index 1b03f70f..1a6e5247 100644 --- a/bifacial_radiance/mismatch.py +++ b/bifacial_radiance/mismatch.py @@ -170,7 +170,8 @@ def mad_fn(data): Parameters ---------- data : np.ndarray - Gtotal irradiance measurements. + Gtotal irradiance measurements. If data is a pandas.DataFrame, one + MAD/Average is returned for each index, based on values across columns. Returns ------- @@ -181,11 +182,17 @@ def mad_fn(data): ''' import numpy as np import pandas as pd - # Pandas returns a notimplemented error if this is a series. - if type(data) == pd.Series: + def _mad_fn(data): + return (np.abs(np.subtract.outer(data,data)).sum()/float(data.__len__())**2 / np.mean(data))*100 + # Pandas returns a notimplemented error if this is a DataFrame. + if (type(data) == pd.Series): data = data.to_numpy() - return (np.abs(np.subtract.outer(data,data)).sum()/float(data.__len__())**2 / np.mean(data))*100 + if type(data) == pd.DataFrame: + temp = data.apply(pd.Series.to_numpy, axis=1) + return(temp.apply(_mad_fn)) + else: + return _mad_fn(data) @@ -322,8 +329,8 @@ def analysisIrradianceandPowerMismatch(testfolder, writefiletitle, portraitorlan # Statistics Calculatoins dfst=pd.DataFrame() - dfst['MAD/G_Total'] = mad_fn(Poat.T) - dfst['Front_MAD/G_Total'] = mad_fn(F.T) + dfst['MAD/G_Total'] = mad_fn(Poat) + dfst['Front_MAD/G_Total'] = mad_fn(F) dfst['MAD/G_Total**2'] = dfst['MAD/G_Total']**2 dfst['Front_MAD/G_Total**2'] = dfst['Front_MAD/G_Total']**2 dfst['poat'] = Poat.mean(axis=1) diff --git a/tests/test_mismatch.py b/tests/test_mismatch.py index ead38f5f..07d36ac2 100644 --- a/tests/test_mismatch.py +++ b/tests/test_mismatch.py @@ -54,13 +54,16 @@ def test_MAD(): pytest.approx(2433.333,abs = 0.001) temp = bifacial_radiance.mismatch.mad_fn(pd.DataFrame(TEST_ARRAY)) - ans = pd.Series([15706.061,4936.190,2928.249,2081.526,1614.642,1318.8295]) + #ans = pd.Series([15706.061,4936.190,2928.249,2081.526,1614.642,1318.8295]) + ans = pd.Series([ 70.892019, 69.953052, 69.014085, 68.075117, + 67.136150, 66.197183, 65.258216, 64.319249, + 63.380282, 62.441315, 61.502347, 60.563380]) pd.testing.assert_series_equal(temp,ans,check_less_precise=True) # test pd.Series objects are correctly handled assert bifacial_radiance.mismatch.mad_fn(ans) == \ - pytest.approx(96.491,abs = 0.001) -# assert temp == \ -# pytest.approx(2433.333,abs = 0.001) + pytest.approx(5.674, abs = 0.001) + #pytest.approx(96.491,abs = 0.001) + def test_analysisIrradianceandPowerMismatch(): #analysisIrradianceandPowerMismatch(testfolder, writefiletitle, From f5e13bf21adf90e45e1e27d4d40ef6dbb93ea4e2 Mon Sep 17 00:00:00 2001 From: cdeline Date: Tue, 14 Mar 2023 07:33:15 -0600 Subject: [PATCH 2/6] mismatch.mad_fn updated with new `axis` parameter. Default = 0 (index). If a 2D array or dataframe is passed in, MAD is calculated for each row (or column if `index=1` is used). Addresses the mad_fn errors in #449 --- bifacial_radiance/mismatch.py | 33 +++++++++++++++++++------ docs/sphinx/source/whatsnew/pending.rst | 2 +- tests/test_mismatch.py | 22 +++++++++++------ 3 files changed, 41 insertions(+), 16 deletions(-) diff --git a/bifacial_radiance/mismatch.py b/bifacial_radiance/mismatch.py index 1a6e5247..34e03959 100644 --- a/bifacial_radiance/mismatch.py +++ b/bifacial_radiance/mismatch.py @@ -163,36 +163,53 @@ def calculatePVMismatch(pvsys, stdpl, cellsx, cellsy, Gpoat): return PowerAveraged, PowerDetailed -def mad_fn(data): +def mad_fn(data, axis='index'): ''' Mean average deviation calculation for mismatch purposes. Parameters ---------- - data : np.ndarray + data : np.ndarray or pd.Series or pd.DataFrame Gtotal irradiance measurements. If data is a pandas.DataFrame, one MAD/Average is returned for each index, based on values across columns. + + axis : {0 or 'index', 1 or 'columns'}, default 'index' + Calculate mean average deviation across rows (default) or columns for 2D data + * 0, or 'index' : MAD calculated across rows. + * 1, or 'columns' : MAD calculated across columns. Returns ------- - scalar : return MAD / Average for a 1D array + scalar or pd.Series: return MAD / Average [%]. Scalar for a 1D array, Series for 2D. + - Equation: 1/(n^2*Gavg)*Sum Sum (abs(G_i - G_j)) - ## Note: starting with Pandas 1.0.0 this function will not work on Series objects. + Equation: 1/(n^2*Gavg)*Sum Sum (abs(G_i - G_j)) * 100[%] + ''' import numpy as np import pandas as pd - def _mad_fn(data): + def _mad_1D(data): #1D calculation of MAD return (np.abs(np.subtract.outer(data,data)).sum()/float(data.__len__())**2 / np.mean(data))*100 + if type(axis) == str: + try: + axis = {"index": 0, "rows": 0, 'columns':1}[axis] + except KeyError: + raise Exception('Incorrect index string in mad_fn. options: index, rows, columns.') + + ndim = data.ndim + if ndim == 2 and axis==1: + data = data.T # Pandas returns a notimplemented error if this is a DataFrame. if (type(data) == pd.Series): data = data.to_numpy() if type(data) == pd.DataFrame: temp = data.apply(pd.Series.to_numpy, axis=1) - return(temp.apply(_mad_fn)) + return(temp.apply(_mad_1D)) + elif ndim ==2: #2D array + return [_mad_1D(i) for i in data] else: - return _mad_fn(data) + return _mad_1D(data) diff --git a/docs/sphinx/source/whatsnew/pending.rst b/docs/sphinx/source/whatsnew/pending.rst index 3e3ba80d..2b036dd8 100644 --- a/docs/sphinx/source/whatsnew/pending.rst +++ b/docs/sphinx/source/whatsnew/pending.rst @@ -10,7 +10,7 @@ API Changes *A new function can now be called to compile results and report out final irradiance and performance data: :py:class:`~bifacial_radiance.RadianceObj.compileResults`. *Multiple modules and rows can now be selected in a single analysis scan. ``modWanted`` and ``rowWanted`` inputs in :py:class:`~bifacial_radiance.RadianceObj.analysis1axis` can now be a list, to select multiple rows and modules for scans. (:issue:`405`)(:pull:`408`) *To support multiple modules and row scans for 1axis simulations, outputs like Wm2Front are now stored in ``trackerdict``.``Results`` (:issue:`405`)(:pull:`408`) - +* ``mismatch.mad_fn`` has new functionality and input parameter `axis`. If a 2D matrix or dataframe is passed in as data, MAD is calculated for each row (default) or column by passing 'axis=1' Enhancements ~~~~~~~~~~~~ diff --git a/tests/test_mismatch.py b/tests/test_mismatch.py index 07d36ac2..6812cc4c 100644 --- a/tests/test_mismatch.py +++ b/tests/test_mismatch.py @@ -49,20 +49,28 @@ def test_setupforPVMismatch(): def test_MAD(): + ans = pd.Series([ 70.892019, 69.953052, 69.014085, 68.075117, + 67.136150, 66.197183, 65.258216, 64.319249, + 63.380282, 62.441315, 61.502347, 60.563380]) + ans_T = pd.Series([72.222222, 22.698413, 13.465160, + 9.571620, 7.424714, 6.064461]) assert bifacial_radiance.mismatch.mad_fn(TEST_ARRAY) == \ - pytest.approx(2433.333,abs = 0.001) + pytest.approx(ans.to_numpy(), abs = 0.001) + + assert bifacial_radiance.mismatch.mad_fn(TEST_ARRAY, axis='columns') == \ + pytest.approx(ans_T.to_numpy(), abs = 0.001) temp = bifacial_radiance.mismatch.mad_fn(pd.DataFrame(TEST_ARRAY)) - #ans = pd.Series([15706.061,4936.190,2928.249,2081.526,1614.642,1318.8295]) - ans = pd.Series([ 70.892019, 69.953052, 69.014085, 68.075117, - 67.136150, 66.197183, 65.258216, 64.319249, - 63.380282, 62.441315, 61.502347, 60.563380]) - pd.testing.assert_series_equal(temp,ans,check_less_precise=True) + temp2 = bifacial_radiance.mismatch.mad_fn(pd.DataFrame(TEST_ARRAY), axis=1) + pd.testing.assert_series_equal(temp, ans) + pd.testing.assert_series_equal(temp2, ans_T) # test pd.Series objects are correctly handled assert bifacial_radiance.mismatch.mad_fn(ans) == \ pytest.approx(5.674, abs = 0.001) - #pytest.approx(96.491,abs = 0.001) + assert bifacial_radiance.mismatch.mad_fn(ans.to_numpy()) == \ + pytest.approx(5.674, abs = 0.001) + def test_analysisIrradianceandPowerMismatch(): From d432ae04b7a7c71fb65deaa34435679e18428b43 Mon Sep 17 00:00:00 2001 From: cdeline Date: Wed, 15 Mar 2023 14:08:46 -0600 Subject: [PATCH 3/6] pytests for mismatch_fit3 with new mad_fn changes --- bifacial_radiance/mismatch.py | 39 +++++++++++++++++++++++++++++++++++ tests/test_mismatch.py | 7 +++++++ 2 files changed, 46 insertions(+) diff --git a/bifacial_radiance/mismatch.py b/bifacial_radiance/mismatch.py index 34e03959..9cd7fb40 100644 --- a/bifacial_radiance/mismatch.py +++ b/bifacial_radiance/mismatch.py @@ -163,6 +163,45 @@ def calculatePVMismatch(pvsys, stdpl, cellsx, cellsy, Gpoat): return PowerAveraged, PowerDetailed +def mismatch_fit3(data): + ''' + Electrical mismatch calculation following Progress in PV paper + Estimating and parameterizing mismatch power loss in bifacial photovoltaic systems + Chris Deline, Silvana Ayala Pelaez,Sara MacAlpine,Carlos Olalla + https://doi.org/10.1002/pip.3259 + + Parameters + ---------- + data : np.ndarray, pd.Series, pd.DataFrame + Gtotal irradiance measurements. Each column is the irradiance for a module + at a specific time. + + Returns + ------- + fit3 : Float or pd.Series + Returns mismatch values for each module + + Equation: 1/(n^2*Gavg)*Sum Sum (abs(G_i - G_j)) + ## Note: starting with Pandas 1.0.0 this function will not work on Series objects. + ''' + import numpy as np + import pandas as pd + + if type(data) == np.ndarray: + data = pd.DataFrame(data) + + datac = data[~np.isnan(data)] + mad = mad_fn(datac, axis=1) /100 # (percentage) + mad2 = mad**2 + + fit3 = 0.054*mad + 0.068*mad2 + + if fit3.__len__() == 1: + fit3 = float(fit3) + + return fit3 + + def mad_fn(data, axis='index'): ''' Mean average deviation calculation for mismatch purposes. diff --git a/tests/test_mismatch.py b/tests/test_mismatch.py index 6812cc4c..8e5fd218 100644 --- a/tests/test_mismatch.py +++ b/tests/test_mismatch.py @@ -88,4 +88,11 @@ def test_analysisIrradianceandPowerMismatch(): df_all = pd.read_csv(writefiletitle) assert df_all.Mismatch_rel[0] == pytest.approx(0.376, abs = 0.001) assert df_all["MAD/G_Total"][0] == pytest.approx(1.987, abs = 0.001) + + +def test_mismatch_fit3(): + ans = pd.Series([0.074469, 0.015761, 0.008504, 0.005792, 0.004384, 0.003525]) + pd.testing.assert_series_equal( bifacial_radiance.mismatch.mismatch_fit3(TEST_ARRAY), ans, atol=1e-6) + pd.testing.assert_series_equal( bifacial_radiance.mismatch.mismatch_fit3(pd.DataFrame(TEST_ARRAY)), ans, atol=1e-6) + assert bifacial_radiance.mismatch.mismatch_fit3(TEST_ARRAY[:,0]) == pytest.approx(ans[0], abs = 0.001) \ No newline at end of file From 4853d26d1bc8c2c7c86b45a771e48e3f7b4576c1 Mon Sep 17 00:00:00 2001 From: cdeline Date: Thu, 9 Nov 2023 12:48:00 -0700 Subject: [PATCH 4/6] Change orientation of mad_fn action to match pandas default. Fixes #449 --- bifacial_radiance/main.py | 2 +- bifacial_radiance/mismatch.py | 8 ++++---- setup.py | 2 +- tests/test_mismatch.py | 8 ++++---- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/bifacial_radiance/main.py b/bifacial_radiance/main.py index 56cbce9e..c148df2e 100644 --- a/bifacial_radiance/main.py +++ b/bifacial_radiance/main.py @@ -264,7 +264,7 @@ def _subhourlydatatoGencumskyformat(gencumskydata, label='right'): #Resample to hourly. Gencumsky wants right-labeled data. - gencumskydata = gencumskydata.resample('60T', closed='right', label='right').mean() + gencumskydata = gencumskydata.resample('60T', closed='right', label='right').mean(numeric_only=True) if label == 'left': #switch from left to right labeled by adding an hour gencumskydata.index = gencumskydata.index + pd.to_timedelta('1H') diff --git a/bifacial_radiance/mismatch.py b/bifacial_radiance/mismatch.py index 9cd7fb40..1bfb0a2c 100644 --- a/bifacial_radiance/mismatch.py +++ b/bifacial_radiance/mismatch.py @@ -191,7 +191,7 @@ def mismatch_fit3(data): data = pd.DataFrame(data) datac = data[~np.isnan(data)] - mad = mad_fn(datac, axis=1) /100 # (percentage) + mad = mad_fn(datac) /100 # (percentage) mad2 = mad**2 fit3 = 0.054*mad + 0.068*mad2 @@ -236,7 +236,7 @@ def _mad_1D(data): #1D calculation of MAD raise Exception('Incorrect index string in mad_fn. options: index, rows, columns.') ndim = data.ndim - if ndim == 2 and axis==1: + if ndim == 2 and axis==0: data = data.T # Pandas returns a notimplemented error if this is a DataFrame. if (type(data) == pd.Series): @@ -385,8 +385,8 @@ def analysisIrradianceandPowerMismatch(testfolder, writefiletitle, portraitorlan # Statistics Calculatoins dfst=pd.DataFrame() - dfst['MAD/G_Total'] = mad_fn(Poat) - dfst['Front_MAD/G_Total'] = mad_fn(F) + dfst['MAD/G_Total'] = mad_fn(Poat.T) + dfst['Front_MAD/G_Total'] = mad_fn(F.T) dfst['MAD/G_Total**2'] = dfst['MAD/G_Total']**2 dfst['Front_MAD/G_Total**2'] = dfst['Front_MAD/G_Total']**2 dfst['poat'] = Poat.mean(axis=1) diff --git a/setup.py b/setup.py index 214a8e88..650f2343 100644 --- a/setup.py +++ b/setup.py @@ -96,7 +96,7 @@ # requirements files see: # https://packaging.python.org/en/latest/requirements.html install_requires=[ - 'pandas < 2.0 ', + 'pandas ', 'pvlib >= 0.8.0', 'pvmismatch', 'configparser', diff --git a/tests/test_mismatch.py b/tests/test_mismatch.py index 8e5fd218..fbc4a9f6 100644 --- a/tests/test_mismatch.py +++ b/tests/test_mismatch.py @@ -49,10 +49,10 @@ def test_setupforPVMismatch(): def test_MAD(): - ans = pd.Series([ 70.892019, 69.953052, 69.014085, 68.075117, + ans_T = pd.Series([ 70.892019, 69.953052, 69.014085, 68.075117, 67.136150, 66.197183, 65.258216, 64.319249, 63.380282, 62.441315, 61.502347, 60.563380]) - ans_T = pd.Series([72.222222, 22.698413, 13.465160, + ans = pd.Series([72.222222, 22.698413, 13.465160, 9.571620, 7.424714, 6.064461]) assert bifacial_radiance.mismatch.mad_fn(TEST_ARRAY) == \ @@ -66,9 +66,9 @@ def test_MAD(): pd.testing.assert_series_equal(temp, ans) pd.testing.assert_series_equal(temp2, ans_T) # test pd.Series objects are correctly handled - assert bifacial_radiance.mismatch.mad_fn(ans) == \ + assert bifacial_radiance.mismatch.mad_fn(ans_T) == \ pytest.approx(5.674, abs = 0.001) - assert bifacial_radiance.mismatch.mad_fn(ans.to_numpy()) == \ + assert bifacial_radiance.mismatch.mad_fn(ans_T.to_numpy()) == \ pytest.approx(5.674, abs = 0.001) From 4b7e30ec02a6ea511fad8238b3cea717bdd5c16e Mon Sep 17 00:00:00 2001 From: cdeline Date: Thu, 9 Nov 2023 13:12:23 -0700 Subject: [PATCH 5/6] minor cleanup to avoid multiple transpositions --- bifacial_radiance/mismatch.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/bifacial_radiance/mismatch.py b/bifacial_radiance/mismatch.py index 1bfb0a2c..cfb58e08 100644 --- a/bifacial_radiance/mismatch.py +++ b/bifacial_radiance/mismatch.py @@ -377,24 +377,19 @@ def analysisIrradianceandPowerMismatch(testfolder, writefiletitle, portraitorlan F.index='FrontIrradiance_cell_'+F.index.astype(str) B.index='BackIrradiance_cell_'+B.index.astype(str) Poat.index='POAT_Irradiance_cell_'+Poat.index.astype(str) - - ## Transpose - F = F.T - B = B.T - Poat = Poat.T # Statistics Calculatoins dfst=pd.DataFrame() - dfst['MAD/G_Total'] = mad_fn(Poat.T) - dfst['Front_MAD/G_Total'] = mad_fn(F.T) + dfst['MAD/G_Total'] = mad_fn(Poat) + dfst['Front_MAD/G_Total'] = mad_fn(F) dfst['MAD/G_Total**2'] = dfst['MAD/G_Total']**2 dfst['Front_MAD/G_Total**2'] = dfst['Front_MAD/G_Total']**2 - dfst['poat'] = Poat.mean(axis=1) - dfst['gfront'] = F.mean(axis=1) - dfst['grear'] = B.mean(axis=1) + dfst['poat'] = Poat.mean() + dfst['gfront'] = F.mean() + dfst['grear'] = B.mean() dfst['bifi_ratio'] = dfst['grear']/dfst['gfront'] - dfst['stdev'] = Poat.std(axis=1)/ dfst['poat'] - dfst.index=Poat.index.astype(str) + dfst['stdev'] = Poat.std()/ dfst['poat'] + dfst.index=Poat.columns.astype(str) # Power Calculations/Saving Pout=pd.DataFrame() @@ -404,10 +399,10 @@ def analysisIrradianceandPowerMismatch(testfolder, writefiletitle, portraitorlan Pout['Front_Pdet']=Pdet_front_all Pout['Mismatch_rel'] = 100-(Pout['Pdet']*100/Pout['Pavg']) Pout['Front_Mismatch_rel'] = 100-(Pout['Front_Pdet']*100/Pout['Front_Pavg']) - Pout.index=Poat.index.astype(str) + Pout.index=Poat.columns.astype(str) - ## Save CSV - df_all = pd.concat([Pout,dfst,Poat,F,B],axis=1) + ## Save CSV as one long row + df_all = pd.concat([Pout, dfst, Poat.T, F.T, B.T], axis=1) df_all.to_csv(writefiletitle) print("Saved Results to ", writefiletitle) From b92a1b433ca41d07afa451ea2b54ca8bc9ad909c Mon Sep 17 00:00:00 2001 From: cdeline Date: Thu, 9 Nov 2023 13:37:22 -0700 Subject: [PATCH 6/6] handle exception for Pandas < 2.0. Update whatsnew/pending.rst --- bifacial_radiance/main.py | 5 ++++- docs/sphinx/source/whatsnew/pending.rst | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/bifacial_radiance/main.py b/bifacial_radiance/main.py index c148df2e..574ca27c 100644 --- a/bifacial_radiance/main.py +++ b/bifacial_radiance/main.py @@ -264,7 +264,10 @@ def _subhourlydatatoGencumskyformat(gencumskydata, label='right'): #Resample to hourly. Gencumsky wants right-labeled data. - gencumskydata = gencumskydata.resample('60T', closed='right', label='right').mean(numeric_only=True) + try: + gencumskydata = gencumskydata.resample('60T', closed='right', label='right').mean() + except TypeError: # Pandas 2.0 error + gencumskydata = gencumskydata.resample('60T', closed='right', label='right').mean(numeric_only=True) if label == 'left': #switch from left to right labeled by adding an hour gencumskydata.index = gencumskydata.index + pd.to_timedelta('1H') diff --git a/docs/sphinx/source/whatsnew/pending.rst b/docs/sphinx/source/whatsnew/pending.rst index 2b036dd8..b0542c83 100644 --- a/docs/sphinx/source/whatsnew/pending.rst +++ b/docs/sphinx/source/whatsnew/pending.rst @@ -10,7 +10,7 @@ API Changes *A new function can now be called to compile results and report out final irradiance and performance data: :py:class:`~bifacial_radiance.RadianceObj.compileResults`. *Multiple modules and rows can now be selected in a single analysis scan. ``modWanted`` and ``rowWanted`` inputs in :py:class:`~bifacial_radiance.RadianceObj.analysis1axis` can now be a list, to select multiple rows and modules for scans. (:issue:`405`)(:pull:`408`) *To support multiple modules and row scans for 1axis simulations, outputs like Wm2Front are now stored in ``trackerdict``.``Results`` (:issue:`405`)(:pull:`408`) -* ``mismatch.mad_fn`` has new functionality and input parameter `axis`. If a 2D matrix or dataframe is passed in as data, MAD is calculated for each row (default) or column by passing 'axis=1' +* ``mismatch.mad_fn`` has new functionality and input parameter `axis`. If a 2D matrix or dataframe is passed in as data, MAD is calculated along the row (default) or along the columns by passing 'axis=1' Enhancements ~~~~~~~~~~~~ @@ -18,7 +18,7 @@ Enhancements Bug fixes ~~~~~~~~~ -* Pandas 2.0 errors have not yet been fixed. Constraining pandas < 2.0 in setup.py until (:issue:`449`) is fixed +* Fixed Pandas 2.0 errors by re-factoring ``mismatch.mad_fn`` (:issue:`449`) Documentation ~~~~~~~~~~~~~~