Commit 61d55de4 authored by William Monroe's avatar William Monroe
Browse files

stripping notebook content

parent c7f21bd8
[core] [core]
attributesfile = ~/.gitattributes_global attributesfile = ~/.gitattributes
[filter "nbstrip_full"] [filter "nbstrip_full"]
clean = "jq --indent 1 \ clean = "jq --indent 1 \
'(.cells[] | select(has(\"outputs\")) | .outputs) = [] \ '(.cells[] | select(has(\"outputs\")) | .outputs) = [] \
......
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
import numpy as np import numpy as np
import pandas as pd import pandas as pd
import pandas_profiling import pandas_profiling
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
df = pd.read_csv('userusage.txt',delimiter='|') df = pd.read_csv('userusage.txt',delimiter='|')
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
df.head() df.head()
``` ```
%% Output %% Output
User Start JobID JobName State \ User Start JobID JobName State \
0 user 2019-01-06T22:00:21 2040834 _interactive COMPLETED 0 user 2019-01-06T22:00:21 2040834 _interactive COMPLETED
1 NaN 2019-01-06T22:00:21 2040834.batch batch COMPLETED 1 NaN 2019-01-06T22:00:21 2040834.batch batch COMPLETED
2 user 2019-01-07T16:15:21 2043373 Pipe_trim_galore COMPLETED 2 user 2019-01-07T16:15:21 2043373 Pipe_trim_galore COMPLETED
3 NaN 2019-01-07T16:15:21 2043373.batch batch COMPLETED 3 NaN 2019-01-07T16:15:21 2043373.batch batch COMPLETED
4 user 2019-01-07T16:15:21 2043374 Pipe_trim_galore COMPLETED 4 user 2019-01-07T16:15:21 2043374 Pipe_trim_galore COMPLETED
Partition MaxRSS ReqMem ReqCPUS NodeList NNodes Elapsed Partition MaxRSS ReqMem ReqCPUS NodeList NNodes Elapsed
0 medium NaN 10000Mc 1 c0088 1 16:04:23 0 medium NaN 10000Mc 1 c0088 1 16:04:23
1 NaN 1394528K 10000Mc 1 c0088 1 16:04:23 1 NaN 1394528K 10000Mc 1 c0088 1 16:04:23
2 medium NaN 2000Mc 1 c0038 1 00:18:41 2 medium NaN 2000Mc 1 c0038 1 00:18:41
3 NaN 58592K 2000Mc 1 c0038 1 00:18:41 3 NaN 58592K 2000Mc 1 c0038 1 00:18:41
4 medium NaN 2000Mc 1 c0063 1 00:15:48 4 medium NaN 2000Mc 1 c0063 1 00:15:48
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
df[['jid','step']] = df.JobID.str.split(".",expand=True) df[['jid','step']] = df.JobID.str.split(".",expand=True)
df.Partition.values df.Partition.values
``` ```
%% Output %% Output
array(['medium', nan, 'medium', ..., 'medium', nan, nan], dtype=object) array(['medium', nan, 'medium', ..., 'medium', nan, nan], dtype=object)
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
batchDF=df.dropna(subset=["MaxRSS"]) batchDF=df.dropna(subset=["MaxRSS"])
userDF=df.dropna(subset=["User"]) userDF=df.dropna(subset=["User"])
for jid in df.jid.unique(): for jid in df.jid.unique():
userDF['MaxRSS'][userDF['jid'] == jid]=batchDF['MaxRSS'][batchDF['jid'] == jid] userDF['MaxRSS'][userDF['jid'] == jid]=batchDF['MaxRSS'][batchDF['jid'] == jid]
#print(userDF[userDF['jid'] == jid]) #print(userDF[userDF['jid'] == jid])
userDF.head() userDF.head()
``` ```
%% Output %% Output
/home/wsmonroe/.conda/envs/wsmplayground/lib/python3.6/site-packages/ipykernel_launcher.py:4: SettingWithCopyWarning: /home/wsmonroe/.conda/envs/wsmplayground/lib/python3.6/site-packages/ipykernel_launcher.py:4: SettingWithCopyWarning:
A value is trying to be set on a copy of a slice from a DataFrame A value is trying to be set on a copy of a slice from a DataFrame
See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
after removing the cwd from sys.path. after removing the cwd from sys.path.
/home/wsmonroe/.conda/envs/wsmplayground/lib/python3.6/site-packages/pandas/core/generic.py:7626: SettingWithCopyWarning: /home/wsmonroe/.conda/envs/wsmplayground/lib/python3.6/site-packages/pandas/core/generic.py:7626: SettingWithCopyWarning:
A value is trying to be set on a copy of a slice from a DataFrame A value is trying to be set on a copy of a slice from a DataFrame
See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
self._update_inplace(new_data) self._update_inplace(new_data)
/home/wsmonroe/.conda/envs/wsmplayground/lib/python3.6/site-packages/IPython/core/interactiveshell.py:2961: SettingWithCopyWarning: /home/wsmonroe/.conda/envs/wsmplayground/lib/python3.6/site-packages/IPython/core/interactiveshell.py:2961: SettingWithCopyWarning:
A value is trying to be set on a copy of a slice from a DataFrame A value is trying to be set on a copy of a slice from a DataFrame
See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy
exec(code_obj, self.user_global_ns, self.user_ns) exec(code_obj, self.user_global_ns, self.user_ns)
%% Cell type:markdown id: tags:
# add more graphs here
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
``` ```
......
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
import sqlite3 import sqlite3
import slurm2sql import slurm2sql
import pandas as pd import pandas as pd
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
db = sqlite3.connect('test.db') db = sqlite3.connect('test.db')
slurm2sql.slurm2sql(db, ['-S', '2020-03-18', '-a']) slurm2sql.slurm2sql(db, ['-S', '2020-03-18', '-a'])
``` ```
%% Output %% Output
0 0
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
# For example, you can then convert to a dataframe: # For example, you can then convert to a dataframe:
df1 = pd.read_sql('SELECT * FROM slurm', db) df1 = pd.read_sql('SELECT * FROM slurm', db)
``` ```
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
df1.head(5) df1.head(5)
``` ```
%% Output %% Output
JobID ArrayJobID ArrayTaskID JobStep JobIDSlurm \ JobID ArrayJobID ArrayTaskID JobStep JobIDSlurm \
0 3319116 3319116 NaN None 3319116_[43-45,47%5] 0 3319116 3319116 NaN None 3319116_[43-45,47%5]
1 3927198 3887451 30.0 None 3887451_30 1 3927198 3887451 30.0 None 3887451_30
2 3927198 3887451 30.0 batch 3887451_30.batch 2 3927198 3887451 30.0 batch 3887451_30.batch
3 3927198 3887451 30.0 extern 3887451_30.extern 3 3927198 3887451 30.0 extern 3887451_30.extern
4 3927199 3887451 31.0 None 3887451_31 4 3927199 3887451 31.0 None 3887451_31
JobName User Group Account State ... \ JobName User Group Account State ... \
0 1mUD1MPa user user user PENDING ... 0 1mUD1MPa user user user PENDING ...
1 100kCrC20MPa user user user COMPLETED ... 1 100kCrC20MPa user user user COMPLETED ...
2 batch user COMPLETED ... 2 batch user COMPLETED ...
3 extern user COMPLETED ... 3 extern user COMPLETED ...
4 100kCrC20MPa user user user COMPLETED ... 4 100kCrC20MPa user user user COMPLETED ...
MaxDiskReadNode MaxDiskReadTask MaxDiskWrite MaxDiskWriteNode \ MaxDiskReadNode MaxDiskReadTask MaxDiskWrite MaxDiskWriteNode \
0 NaN 0 NaN
1 NaN 1 NaN
2 c0088 0 1.222336e+10 c0088 2 c0088 0 1.222336e+10 c0088
3 c0088 0 0.000000e+00 c0088 3 c0088 0 0.000000e+00 c0088
4 NaN 4 NaN
MaxDiskWriteTask ReqGPUS Comment GPUMem GPUEff NGPU MaxDiskWriteTask ReqGPUS Comment GPUMem GPUEff NGPU
0 NaN None None None None 0 NaN None None None None
1 NaN None None None None 1 NaN None None None None
2 0 NaN None None None None 2 0 NaN None None None None
3 0 NaN None None None None 3 0 NaN None None None None
4 NaN None None None None 4 NaN None None None None
[5 rows x 63 columns] [5 rows x 63 columns]
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
print("more plots to come")
```
%% Output
more plots to come
%% Cell type:code id: tags:
``` python
``` ```
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment