initial package made. data query to CSV via commandline script and python functions

*.ini

*.pyc

__pycache__

*.ipynb

.vscode/

 No newline at end of file

from .data import get_trend

import sys

from argparse import ArgumentParser, RawDescriptionHelpFormatter

from dateutil.parser import parse

from .data import get_trend

parser = ArgumentParser(

    prog='bdx',

    # usage='python -m bdx user password trend start end [file.csv]',

    description=('Download data from Building Logix Data Exchange (BDX) '

                 'and save it to csv.'),

    epilog=('Date/time formats can be\n'

            ' - 2020-02-03T15:30:00Z-6 (i.e. Feb 3 2020 3:30 PM UTC-6 timezone)\n'

            ' - 20200203T153000Z-6\n'),

    formatter_class=RawDescriptionHelpFormatter)

parser.add_argument('user', help='Username for BDX', type=str)

parser.add_argument('password', help='Password for BDX', type=str)

parser.add_argument('trend', help='ID of trend to download', type=str)

parser.add_argument('start', help='Start time in ISO 8601 format.', type=parse)

parser.add_argument('end', help='Start time in ISO 8601 format.', type=parse)

parser.add_argument('--file', help=('Filenname to save CSV. If not provided, '

    'printed to console'), type=str, default=sys.stdout)

args = parser.parse_args()

trend = get_trend(trend_id=args.trend,

                  username=args.user,

                  password=args.password,

                  start=args.start,

                  end=args.end)

trend.to_csv(args.file)

"""

Functions for obtaining data from various sources.

"""

from typing import List, Dict, Any

from datetime import datetime

from urllib.parse import urlencode

import json

import pandas as pd

import numpy as np

from bs4 import BeautifulSoup

import requests

def get_trend(trend_id: str, username: str, password: str, start: datetime,

    end: datetime, aggregation: str='Point') -> pd.DataFrame:

    """

    Get a pandas DataFrame of a trend on BuildingLogix Data Exchange (BDX).

    Parameters

    ----------

    trend_id : str

        The id of the trend (numeric form as a string).

    username : str

        Username to access BDX.

    password : str

        Password to access BDX.

    start : datetime

        datetime.datetime object. If no timezone specified, converted to UTC.

    end : datetime

        datetime.datetime object. If no timezone specified, converted to UTC.

    aggregation : str, optional

        Aggregation level for trend, by default 'Point'

    Returns

    -------

    pd.DataFrame

        A DataFrame with datetime as index, and columns for each time series in

        the trend. NaN values indicate missing data.

    """

    baseURL = 'https://facilities.app.vanderbilt.edu/'

    loginURL = baseURL + 'bdx/'

    authURL = loginURL + 'faces/login.xhtml'

    successURL = loginURL + 'loginsuccess.xhtml'

    trendinfoURL = baseURL + 'trendview/api/trend-info/' + str(trend_id)

    trendvaluesURL = baseURL + 'trendview/api/data-values'

    sess = requests.Session()

    xml = BeautifulSoup(sess.get(loginURL).content, 'html.parser')

    view_state = None

    for i in xml.find_all('input'):

        if i.get('name') == 'javax.faces.ViewState':

            view_state = i.get('value')

    payload = {

        'login': 'login',

        'login:j_username': username,

        'login:j_password': password,

        'login:autoLogin': False,

        'login:loginSubmit': 'Login',

        'javax.faces.ViewState': view_state}

    res_auth = sess.post(authURL, data=urlencode(payload),

                         headers={'Content-Type': 'application/x-www-form-urlencoded'})

    key = res_auth.cookies['IS_SSO']

    res_trend_info = sess.get(trendinfoURL)

    trend_info = json.loads(res_trend_info.content)

    payload = {

        'selectorList': trend_info['values'],

        'startDate': start.isoformat(),

        'endDate': end.isoformat(),

        'aggregationLevel': aggregation

    }

    r = sess.post(trendvaluesURL, json=payload)

    trend = parse_trend_dict(json.loads(r.content))

    return trend

def parse_trend_dict(trend_dict: List[Dict[str, Any]]) -> pd.DataFrame:

    # trend_dict is a list of dictionaries. Each dictionary has the structure:

    # 'propertyName': str, short property name e.g vfdPower

    # 'label': str, qualified property name e.g. 'Cell_1bFan vfdPower'

    # 'trendValueId': int, identifier for field name

    # 'dataValues': time series as a list of dicts, 

    #     [

    #         {'time': str, 'realValue': Any, valueType: ['REAL' or 'NULL']},

    #         {'time': str, 'realValue': Any, valueType: ['REAL' or 'NULL']}

    #         ...

    #         Where:

    #           - time: ISO format 'yyyy-mm-ddThh:mm:ss.sssZ'

    #           - valueType: 'REAL' or 'NULL'. If 'REAL', then 'realValue' key exists

    #           - realValue: The value at that time stamp IF valueType != 'NULL'

    #     ]

    frames = []     # list of dataframes parsed from each trend dictionary

    for time_series in trend_dict:

        series_df = pd.DataFrame(time_series['dataValues'])  # create a DataFrame for each series

        series_df['time'] = pd.to_datetime(series_df['time'])

        series_df.set_index('time', inplace=True)

        # If dataframe does not have a 'realValue' column, it means that the

        # whole series was missing values (valueType='NULL'). So a placeholder

        # column is created with NaN values. These can then be handled on

        # per-column basis in the final DataFrame:

        if 'realValue' not in series_df:

            series_df['realValue'] = np.nan

        series_df.rename(columns={'realValue': time_series['label']}, inplace=True)

        series_df.drop(columns='valueType', inplace=True, errors='ignore')

        frames.append(series_df)

    df = pd.concat(frames, axis='columns', join='outer', sort=True)

    return df

 No newline at end of file