adapted preprocessing to new dataset (2f0486ac) · Commits · Smart Buildings / EngineeringScienceBuilding

docs/5-dataset.md

+4 −0

Changes for docs/5-dataset.md: 4 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		@@ -56,6 +56,10 @@ Each cooling tower/chiller system has the following parameters:

		17. `PerChiLoad`: Cooling load of the chiller as a fraction of maximum electrical capacity. The maximum cooling capacity in tons is 800 tons. The ratio `Tons / 800` should give roughly the same value as `PerChilLoad`.

		18. `FreqFanA`: Spinning rate of fan A in Hertz.

		19. `FreqFanB`: Spinning rate of fan B in Hertz.

		And the following derived fields:

		1. `PowIn`: Total input power calculated as a sum of all power fields.

src/ioops.py

+21 −7

Changes for src/ioops.py: 21 added lines, 7 removed lines.
Original line number	Diff line number	Diff line
		"""
		Input/Output operations for datasets.
		Input/Output operations for datasets stored as excel files. Assumes each excel file
		contains data on a single chiller. Sheets inside a file may contain different sets
		of columns id'd by the same index (for e.g. timestamp).

		Usage:

		* Either import to use functions,
		* Either import to use functions, or

		* Invoke from command line as:

		@@ -13,7 +15,8 @@ python -m ioops [FILE, [FILE,...]]

		To carry out excel to csv conversion.
		"""
		from os.path import abspath, join, dirname
		import os
		from os.path import abspath, join, dirname, splitext, basename

		import pandas as pd
		from dateutil.parser import parse
		@@ -36,18 +39,28 @@ def xlsx_to_csv(xlsx: str):
		Convert an XLSX file to a csv file with proper date-time conversion for faster
		read operations later on.

		* Removes `???` artefacts in cells,
		* Inner joins multiple sheets in excel file on `Time`.

		Args:

		* `xlsx (str)`: The path to the excel file. All sheets are converted to separate
		csvs in the same directory as the excel document.
		"""
		xl_name = splitext(basename(xlsx))[0]
		xl = pd.read_excel(xlsx, sheet_name=None, **ESB_SCHEMA)
		for name, sheet in xl.items():
		sheet = sheet.set_index('Time')
		sheets = [s for _, s in xl.items()]
		for sheet in sheets:
		sheet.set_index('Time', inplace=True)
		# Some cells have '??? ' which is removed to allow for numeric conversion
		for col in sheet.columns:
		sheet[col] = sheet[col].astype(str).str.replace('\?\?\? ', '')
		sheet.to_csv(abspath(join(dirname(xlsx), name + '.csv')))

		aggregate = sheets[0]
		for sheet in sheets[1:]:
		aggregate = aggregate.join(sheet, on='Time', how='inner')

		aggregate.to_csv(abspath(join(dirname(xlsx), xl_name + '.csv')))



		@@ -55,7 +68,8 @@ if __name__ == '__main__':
		import sys
		from glob import glob

		default = [abspath(join(dirname(__file__), '../SystemInfo/BDXChillerData.xlsx'))]
		default = [abspath(join(os.environ.get('DATADIR', './'),
		'EngineeringScienceBuilding', 'Chillers.xlsx'))]
		paths = sys.argv[2:] if len(sys.argv) >= 3 else default
		for arg in paths:
		for xl in glob(arg):

src/preprocess.py

+10 −8

Changes for src/preprocess.py: 10 added lines, 8 removed lines.
Original line number	Diff line number	Diff line
		@@ -28,7 +28,7 @@ TEMP_FIELDS = (
		'TempEvapIn',
		'TempEvapOut',
		'TempAmbient',
		'TempWetbulb'
		'TempWetBulb'
		)
		# Tons fields to convert to watts
		TONS_FIELDS = (
		@@ -49,7 +49,7 @@ PER_FIELDS = (
		'PerFreqFanA',
		'PerFreqFanB',
		'PerHumidity',
		'PerChilLoad'
		'PerChilLoad',
		)
		# Power fields
		POW_FIELDS = (
		@@ -97,20 +97,20 @@ def standardize(df: pd.DataFrame, temp_fields=TEMP_FIELDS, kwatts_fields=KWATTS_

		def fill_missing_temperatures(df: pd.DataFrame) -> pd.DataFrame:
		"""
		Fills missing values for TempAmbient and TempWetbulb
		Fills missing values for TempAmbient and TempWetBulb
		temperature.
		"""
		# Filling in estimates of Wet-Bulb temperature where absent.
		# Requires Ambient temperature and humidity values.
		if 'TempAmbient' in df.columns:
		sel = df['TempWetbulb'].isna()
		df.loc[sel, 'TempWetbulb'] = wetbulb(df.loc[sel, 'TempAmbient'],
		sel = df['TempWetBulb'].isna()
		df.loc[sel, 'TempWetBulb'] = wetbulb(df.loc[sel, 'TempAmbient'],
		df.loc[sel, 'PerHumidity'])
		# Filling in estimates of Ambient temperature where absent.
		# Requires wet-bulb temperature and relative humidity values.
		if 'TempAmbient' in df.columns:
		sel = df['TempAmbient'].isna() & ~df['TempWetbulb'].isna()
		df.loc[sel, 'TempAmbient'] = ambient(df.loc[sel, 'TempWetbulb'],
		sel = df['TempAmbient'].isna() & ~df['TempWetBulb'].isna()
		df.loc[sel, 'TempAmbient'] = ambient(df.loc[sel, 'TempWetBulb'],
		df.loc[sel, 'PerHumidity'])
		return df

		@@ -142,11 +142,13 @@ def calculate_derivative_fields(df: pd.DataFrame):


		if __name__ == '__main__':
		import os
		from os.path import abspath, join, dirname
		import sys
		from glob import glob

		default = [abspath(join(dirname(__file__), '../SystemInfo/*.csv'))]
		default = [abspath(join(os.environ.get('DATADIR', './'),
		'EngineeringScienceBuilding', 'Chillers.csv'))]
		paths = sys.argv[2:] if len(sys.argv) >= 3 else default
		for arg in paths:
		for csv in glob(arg):

Admin message