Skip to content
GitLab
Menu
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
Chirag Chandrahas Shetty
createAndParseSACCT
Commits
f16856f3
Commit
f16856f3
authored
Mar 19, 2020
by
Chirag Chandrahas Shetty
Browse files
Getting slurm data using slurm2sql
parent
eac8f3d5
Changes
2
Hide whitespace changes
Inline
Side-by-side
requirements.txt
0 → 100644
View file @
f16856f3
astroid
==2.3.3
astropy
==4.0
attr
==0.3.1
attrs
==19.3.0
backcall
==0.1.0
bleach
==3.1.3
certifi
==2019.11.28
chardet
==3.0.4
confuse
==1.0.0
cycler
==0.10.0
decorator
==4.4.2
defusedxml
==0.6.0
entrypoints
==0.3
htmlmin
==0.1.12
idna
==2.8
ipykernel
==5.1.4
ipython
==7.13.0
ipython-genutils
==0.2.0
ipywidgets
==7.5.1
isort
==4.3.21
jedi
==0.16.0
Jinja2
==2.11.1
joblib
==0.14.1
jsonschema
==3.2.0
jupyter-client
==6.0.0
jupyter-core
==4.6.1
kaggle
==1.5.6
kiwisolver
==1.1.0
lazy-object-proxy
==1.4.3
llvmlite
==0.31.0
MarkupSafe
==1.1.1
matplotlib
==3.2.0
mccabe
==0.6.1
missingno
==0.4.2
mistune
==0.8.4
more-itertools
==8.2.0
nbconvert
==5.6.1
nbformat
==5.0.4
networkx
==2.4
notebook
==6.0.3
numba
==0.48.0
numpy
==1.18.2
packaging
==20.3
pandas
==0.25.3
pandas-profiling
==2.5.0
pandocfilters
==1.4.2
parso
==0.6.2
pexpect
==4.8.0
phik
==0.9.9
pickleshare
==0.7.5
pluggy
==0.13.1
prometheus-client
==0.7.1
prompt-toolkit
==3.0.3
ptyprocess
==0.6.0
py
==1.8.1
Pygments
==2.6.1
pylint
==2.4.4
pyparsing
==2.4.6
pyrsistent
==0.15.7
pytest
==5.4.1
pytest-pylint
==0.15.1
python-dateutil
==2.8.1
python-slugify
==4.0.0
pytz
==2019.3
PyYAML
==5.3
pyzmq
==18.1.1
requests
==2.22.0
scipy
==1.4.1
seaborn
==0.10.0
Send2Trash
==1.5.0
six
==1.14.0
slurm2sql
==0.9.0
tangled-up-in-unicode
==0.0.3
terminado
==0.8.3
testpath
==0.4.4
text-unidecode
==1.3
tornado
==6.0.4
tqdm
==4.42.0
traitlets
==4.3.3
urllib3
==1.25.8
visions
==0.2.2
wcwidth
==0.1.8
webencodings
==0.5.1
widgetsnbextension
==3.5.1
wrapt
==1.11.2
slurm-2sql.ipynb
0 → 100644
View file @
f16856f3
{
"cells": [
{
"cell_type": "code",
"execution_count": 8,
"metadata": {},
"outputs": [],
"source": [
"import sqlite3\n",
"import slurm2sql\n",
"import pandas as pd"
]
},
{
"cell_type": "code",
"execution_count": 6,
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"0"
]
},
"execution_count": 6,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"db = sqlite3.connect('test.db')\n",
"slurm2sql.slurm2sql(db, ['-S', '2020-03-18', '-a'])"
]
},
{
"cell_type": "code",
"execution_count": 9,
"metadata": {},
"outputs": [],
"source": [
"# For example, you can then convert to a dataframe:\n",
"df1 = pd.read_sql('SELECT * FROM slurm', db)"
]
},
{
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>JobID</th>\n",
" <th>ArrayJobID</th>\n",
" <th>ArrayTaskID</th>\n",
" <th>JobStep</th>\n",
" <th>JobIDSlurm</th>\n",
" <th>JobName</th>\n",
" <th>User</th>\n",
" <th>Group</th>\n",
" <th>Account</th>\n",
" <th>State</th>\n",
" <th>...</th>\n",
" <th>MaxDiskReadNode</th>\n",
" <th>MaxDiskReadTask</th>\n",
" <th>MaxDiskWrite</th>\n",
" <th>MaxDiskWriteNode</th>\n",
" <th>MaxDiskWriteTask</th>\n",
" <th>ReqGPUS</th>\n",
" <th>Comment</th>\n",
" <th>GPUMem</th>\n",
" <th>GPUEff</th>\n",
" <th>NGPU</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>3319116</td>\n",
" <td>3319116</td>\n",
" <td>NaN</td>\n",
" <td>None</td>\n",
" <td>3319116_[43-45,47%5]</td>\n",
" <td>1mUD1MPa</td>\n",
" <td>gpekmezi</td>\n",
" <td>gpekmezi</td>\n",
" <td>gpekmezi</td>\n",
" <td>PENDING</td>\n",
" <td>...</td>\n",
" <td></td>\n",
" <td></td>\n",
" <td>NaN</td>\n",
" <td></td>\n",
" <td></td>\n",
" <td>NaN</td>\n",
" <td>None</td>\n",
" <td>None</td>\n",
" <td>None</td>\n",
" <td>None</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>3927198</td>\n",
" <td>3887451</td>\n",
" <td>30.0</td>\n",
" <td>None</td>\n",
" <td>3887451_30</td>\n",
" <td>100kCrC20MPa</td>\n",
" <td>gpekmezi</td>\n",
" <td>gpekmezi</td>\n",
" <td>gpekmezi</td>\n",
" <td>COMPLETED</td>\n",
" <td>...</td>\n",
" <td></td>\n",
" <td></td>\n",
" <td>NaN</td>\n",
" <td></td>\n",
" <td></td>\n",
" <td>NaN</td>\n",
" <td>None</td>\n",
" <td>None</td>\n",
" <td>None</td>\n",
" <td>None</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>3927198</td>\n",
" <td>3887451</td>\n",
" <td>30.0</td>\n",
" <td>batch</td>\n",
" <td>3887451_30.batch</td>\n",
" <td>batch</td>\n",
" <td></td>\n",
" <td></td>\n",
" <td>gpekmezi</td>\n",
" <td>COMPLETED</td>\n",
" <td>...</td>\n",
" <td>c0088</td>\n",
" <td>0</td>\n",
" <td>1.222336e+10</td>\n",
" <td>c0088</td>\n",
" <td>0</td>\n",
" <td>NaN</td>\n",
" <td>None</td>\n",
" <td>None</td>\n",
" <td>None</td>\n",
" <td>None</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>3927198</td>\n",
" <td>3887451</td>\n",
" <td>30.0</td>\n",
" <td>extern</td>\n",
" <td>3887451_30.extern</td>\n",
" <td>extern</td>\n",
" <td></td>\n",
" <td></td>\n",
" <td>gpekmezi</td>\n",
" <td>COMPLETED</td>\n",
" <td>...</td>\n",
" <td>c0088</td>\n",
" <td>0</td>\n",
" <td>0.000000e+00</td>\n",
" <td>c0088</td>\n",
" <td>0</td>\n",
" <td>NaN</td>\n",
" <td>None</td>\n",
" <td>None</td>\n",
" <td>None</td>\n",
" <td>None</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>3927199</td>\n",
" <td>3887451</td>\n",
" <td>31.0</td>\n",
" <td>None</td>\n",
" <td>3887451_31</td>\n",
" <td>100kCrC20MPa</td>\n",
" <td>gpekmezi</td>\n",
" <td>gpekmezi</td>\n",
" <td>gpekmezi</td>\n",
" <td>COMPLETED</td>\n",
" <td>...</td>\n",
" <td></td>\n",
" <td></td>\n",
" <td>NaN</td>\n",
" <td></td>\n",
" <td></td>\n",
" <td>NaN</td>\n",
" <td>None</td>\n",
" <td>None</td>\n",
" <td>None</td>\n",
" <td>None</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"<p>5 rows × 63 columns</p>\n",
"</div>"
],
"text/plain": [
" JobID ArrayJobID ArrayTaskID JobStep JobIDSlurm \\\n",
"0 3319116 3319116 NaN None 3319116_[43-45,47%5] \n",
"1 3927198 3887451 30.0 None 3887451_30 \n",
"2 3927198 3887451 30.0 batch 3887451_30.batch \n",
"3 3927198 3887451 30.0 extern 3887451_30.extern \n",
"4 3927199 3887451 31.0 None 3887451_31 \n",
"\n",
" JobName User Group Account State ... \\\n",
"0 1mUD1MPa gpekmezi gpekmezi gpekmezi PENDING ... \n",
"1 100kCrC20MPa gpekmezi gpekmezi gpekmezi COMPLETED ... \n",
"2 batch gpekmezi COMPLETED ... \n",
"3 extern gpekmezi COMPLETED ... \n",
"4 100kCrC20MPa gpekmezi gpekmezi gpekmezi COMPLETED ... \n",
"\n",
" MaxDiskReadNode MaxDiskReadTask MaxDiskWrite MaxDiskWriteNode \\\n",
"0 NaN \n",
"1 NaN \n",
"2 c0088 0 1.222336e+10 c0088 \n",
"3 c0088 0 0.000000e+00 c0088 \n",
"4 NaN \n",
"\n",
" MaxDiskWriteTask ReqGPUS Comment GPUMem GPUEff NGPU \n",
"0 NaN None None None None \n",
"1 NaN None None None None \n",
"2 0 NaN None None None None \n",
"3 0 NaN None None None None \n",
"4 NaN None None None None \n",
"\n",
"[5 rows x 63 columns]"
]
},
"execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df1.head(5)"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python [conda env:.conda-slurm-ds]",
"language": "python",
"name": "conda-env-.conda-slurm-ds-py"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.8.1"
}
},
"nbformat": 4,
"nbformat_minor": 2
}
%% Cell type:code id: tags:
```
python
import
sqlite3
import
slurm2sql
import
pandas
as
pd
```
%% Cell type:code id: tags:
```
python
db
=
sqlite3
.
connect
(
'test.db'
)
slurm2sql
.
slurm2sql
(
db
,
[
'-S'
,
'2020-03-18'
,
'-a'
])
```
%% Output
0
%% Cell type:code id: tags:
```
python
# For example, you can then convert to a dataframe:
df1
=
pd
.
read_sql
(
'SELECT * FROM slurm'
,
db
)
```
%% Cell type:code id: tags:
```
python
df1
.
head
(
5
)
```
%% Output
JobID ArrayJobID ArrayTaskID JobStep JobIDSlurm \
0 3319116 3319116 NaN None 3319116_[43-45,47%5]
1 3927198 3887451 30.0 None 3887451_30
2 3927198 3887451 30.0 batch 3887451_30.batch
3 3927198 3887451 30.0 extern 3887451_30.extern
4 3927199 3887451 31.0 None 3887451_31
JobName User Group Account State ... \
0 1mUD1MPa gpekmezi gpekmezi gpekmezi PENDING ...
1 100kCrC20MPa gpekmezi gpekmezi gpekmezi COMPLETED ...
2 batch gpekmezi COMPLETED ...
3 extern gpekmezi COMPLETED ...
4 100kCrC20MPa gpekmezi gpekmezi gpekmezi COMPLETED ...
MaxDiskReadNode MaxDiskReadTask MaxDiskWrite MaxDiskWriteNode \
0 NaN
1 NaN
2 c0088 0 1.222336e+10 c0088
3 c0088 0 0.000000e+00 c0088
4 NaN
MaxDiskWriteTask ReqGPUS Comment GPUMem GPUEff NGPU
0 NaN None None None None
1 NaN None None None None
2 0 NaN None None None None
3 0 NaN None None None None
4 NaN None None None None
[5 rows x 63 columns]
%% Cell type:code id: tags:
```
python
```
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment
