Commit 32cb5a45 authored by John-Paul Robinson's avatar John-Paul Robinson
Browse files

Batch update for multiple changes

Turn off cert verify on query.
Change default date ranges and inspected nodes (still needs to
be more data driven).
Add better logic for checking for saved data.
Add save data logic at end of notebook.
parent 970de6b5
......@@ -86,8 +86,18 @@
"metadata": {},
"outputs": [],
"source": [
"startdate = '2020/01/01 00:00:00'\n",
"enddate = '2021/02/21 00:00:00'"
"startdate = '2021/01/01 00:00:00'\n",
"enddate = '2021/04/8 00:00:00'"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"displaystart = '2021-02-01'\n",
"displaystop = '2021-04-08'"
]
},
{
......@@ -109,7 +119,11 @@
"metadata": {},
"outputs": [],
"source": [
"response = requests.get('https://master:8081/rest/v1/monitoring/dump', params=params, cert=cert, verify=False)"
"if os.path.exists(\"ipower_data.csv\"):\n",
" df = pd.read_csv(\"power_data.csv\")\n",
"else:\n",
" response = requests.get('https://master:8081/rest/v1/monitoring/dump', params=params, cert=cert, verify=False)\n",
" df = pd.DataFrame(response.json()[\"data\"])"
]
},
{
......@@ -119,15 +133,6 @@
"Simply read the json response into a dataframe for futher parsing."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df = pd.DataFrame(response.json()[\"data\"])"
]
},
{
"cell_type": "markdown",
"metadata": {},
......@@ -148,6 +153,15 @@
"#df = df.loc[df['raw'] < 10000]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df"
]
},
{
"cell_type": "markdown",
"metadata": {},
......@@ -192,7 +206,7 @@
"m6_hourly_pwr=pd.DataFrame(np.zeros((1,len(hourly_idx))).T, index=hourly_idx, columns=['sum'])\n",
"\n",
"for num, entity in enumerate(sorted(df.entity.unique())):\n",
" if entity not in ['c0108', 'c0009']:\n",
" if entity not in ['c0009']:\n",
" node_pwr=df[df.entity==entity].set_index(\"datetime\")\n",
" node_pwr=node_pwr[['raw']].resample('H').mean()\n",
" node_pwr=node_pwr[startdate:enddate].fillna(method=\"ffill\")\n",
......@@ -204,11 +218,20 @@
" m6_hourly_pwr[entity]= node_pwr[startdate:enddate]\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"m6_hourly_pwr"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"## Plot Per-node Hourly\n",
"## Plot Per-node Hourly for Row 5 Rack 1\n",
"\n",
"This is just to see the data for each node in one plot and get a feel for how the nodes behave relative to each other. Plot nodes in individual subplotes to decern individual behavior of specific nodes. It does give a sense of how the total power adds up. \n",
"\n",
......@@ -228,7 +251,7 @@
"num_nodes=36\n",
"fig, axes = plt.subplots(num_nodes,1, sharex=True, figsize=(20,30))\n",
"for i in range(num_nodes):\n",
" m6_hourly_pwr['2020-02-01':'2021-02-21'].iloc[:,i+1:i+2].plot(ax=axes[i], legend=True)\n",
" m6_hourly_pwr[displaystart:displaystop].iloc[:,i+1:i+2].plot(ax=axes[i], legend=True)\n",
" axes[i].legend(loc='lower left')"
]
},
......@@ -245,7 +268,61 @@
"metadata": {},
"outputs": [],
"source": [
"m6_hourly_pwr['2020-02-01':'2021-02-21'].iloc[:,1:2].plot()"
"select_node=\"c0022\""
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df[df.entity==select_node].set_index(\"datetime\")[\"2020-09-01\":\"2020-10-04\"]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"m6_hourly_pwr[displaystart:displaystop].iloc[:,3:4][\"2020-10-03\":\"2020-10-04\"]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"m6_hourly_pwr[displaystart:displaystop].iloc[:,3:4][\"2020-09-28\":\"2020-10-14\"].plot()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"m6_hourly_pwr[displaystart:displaystop].iloc[:,3:4].plot()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df[df[\"entity\"]==\"c0001\"]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df[df[\"entity\"]==\"c0001\"][\"datetime\"].max()"
]
},
{
......@@ -263,7 +340,7 @@
"metadata": {},
"outputs": [],
"source": [
"nan_mask = m6_hourly_pwr['2021-02-01':'2021-02-02'].isna()"
"nan_mask = m6_hourly_pwr[\"2021-03-22\":\"2021-03-23\"].isna()"
]
},
{
......@@ -275,6 +352,15 @@
"power_missing = nan_mask[nan_mask].apply(lambda row: row[row == True].index, axis=1)[1]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(*power_missing,sep=\", \")"
]
},
{
"cell_type": "code",
"execution_count": null,
......@@ -288,6 +374,54 @@
" axes[i].legend(loc='lower left')"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"node"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df[df[\"entity\"]==node][\"datetime\"].max()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"lastreport = pd.DataFrame(columns=('node', 'datetime'))\n",
"\n",
"for i, node in enumerate(power_missing):\n",
" lastreport.loc[i] = [node, df[df[\"entity\"]==node][\"datetime\"].max()]"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"lastreport.sort_values(by=\"datetime\") #[\"datetime\"].sort()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"print(\"{}:\\t{}\".format(node, df[df[\"entity\"]==node][\"datetime\"].max()))"
]
},
{
"cell_type": "markdown",
"metadata": {},
......@@ -309,7 +443,7 @@
"for i, node in enumerate(m6_hourly_pwr.iloc[:,1:].columns):\n",
" if (i == num_nodes):\n",
" break\n",
" m6_hourly_pwr[node]['2020-02-01':'2021-02-21'].plot(ax=axes[i], legend=True)\n",
" m6_hourly_pwr[node][displaystart:displaystop].plot(ax=axes[i], legend=True)\n",
" axes[i].legend(loc='lower left')\n"
]
},
......@@ -331,7 +465,7 @@
"metadata": {},
"outputs": [],
"source": [
"kW = m6_hourly_pwr['2020-02-01':'2021-02-21'].sum(axis=1)/1000"
"kW = m6_hourly_pwr[displaystart:displaystop].sum(axis=1)/1000"
]
},
{
......@@ -388,6 +522,33 @@
"ax.set_ylabel('Power (kW)')\n",
"ax.set_title('Cheaha Trends in Electricity Consumption');"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"# Save Hourly Power to Dataframe\n",
"\n",
"This makes it easy to use the data in other analysis and learning efforts."
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"m6_hourly_pwr.to_pickle(\"m6_hourly_pwr.gz\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"df.to_pickle(\"power_stats_raw_df.gz\")"
]
}
],
"metadata": {
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment