Merge branch 'master' into 'master'

Master

See merge request rc-data-science/createandparsesacct!1

environment-slurm-ds.yml

+name: slurm-ds
+channels:
+  - defaults
+dependencies:
+  - _libgcc_mutex=0.1=main.conda
+  - backcall=0.1.0=py38_0.conda
+  - ca-certificates=2020.1.1=0.conda
+  - certifi=2019.11.28=py38_0.conda
+  - decorator=4.4.2=py_0.conda
+  - entrypoints=0.3=py38_0.conda
+  - ipykernel=5.1.4=py38h39e3cac_0.conda
+  - ipython=7.13.0=py38h5ca1d4c_0.conda
+  - ipython_genutils=0.2.0=py38_0.conda
+  - jedi=0.16.0=py38_0.conda
+  - jupyter_client=6.0.0=py_0.conda
+  - jupyter_core=4.6.1=py38_0.conda
+  - ld_impl_linux-64=2.33.1=h53a641e_7.conda
+  - libedit=3.1.20181209=hc058e9b_0.conda
+  - libffi=3.2.1=hd88cf55_4.conda
+  - libgcc-ng=9.1.0=hdf63c60_0.conda
+  - libsodium=1.0.16=h1bed415_0.conda
+  - libstdcxx-ng=9.1.0=hdf63c60_0.conda
+  - ncurses=6.2=he6710b0_0
+  - openssl=1.1.1e=h7b6447c_0.conda
+  - parso=0.6.2=py_0.conda
+  - pexpect=4.8.0=py38_0.conda
+  - pickleshare=0.7.5=py38_1000.conda
+  - pip=20.0.2=py38_1.conda
+  - prompt_toolkit=3.0.3=py_0.conda
+  - ptyprocess=0.6.0=py38_0.conda
+  - pygments=2.6.1=py_0.conda
+  - python=3.8.1=h0371630_1.conda
+  - python-dateutil=2.8.1=py_0
+  - pyzmq=18.1.1=py38he6710b0_0.conda
+  - readline=7.0=h7b6447c_5.conda
+  - setuptools=46.0.0=py38_0.conda
+  - six=1.14.0=py38_0.conda
+  - sqlite=3.31.1=h7b6447c_0.conda
+  - tk=8.6.8=hbc83047_0.conda
+  - tornado=6.0.4=py38h7b6447c_1.conda
+  - traitlets=4.3.3=py38_0.conda
+  - wcwidth=0.1.8=py_0.conda
+  - wheel=0.34.2=py38_0.conda
+  - xz=5.2.4=h14c3975_4.conda
+  - zeromq=4.3.1=he6710b0_3.conda
+  - zlib=1.2.11=h7b6447c_3.conda
+  - pip:
+    - astroid==2.3.3
+    - astropy==4.0
+    - attr==0.3.1
+    - attrs==19.3.0
+    - bleach==3.1.3
+    - chardet==3.0.4
+    - confuse==1.0.0
+    - cycler==0.10.0
+    - defusedxml==0.6.0
+    - htmlmin==0.1.12
+    - idna==2.8
+    - ipywidgets==7.5.1
+    - isort==4.3.21
+    - jinja2==2.11.1
+    - joblib==0.14.1
+    - jsonschema==3.2.0
+    - kaggle==1.5.6
+    - kiwisolver==1.1.0
+    - lazy-object-proxy==1.4.3
+    - llvmlite==0.31.0
+    - markupsafe==1.1.1
+    - matplotlib==3.2.0
+    - mccabe==0.6.1
+    - missingno==0.4.2
+    - mistune==0.8.4
+    - more-itertools==8.2.0
+    - nbconvert==5.6.1
+    - nbformat==5.0.4
+    - networkx==2.4
+    - notebook==6.0.3
+    - numba==0.48.0
+    - numpy==1.18.2
+    - packaging==20.3
+    - pandas==0.25.3
+    - pandas-profiling==2.5.0
+    - pandocfilters==1.4.2
+    - phik==0.9.9
+    - pluggy==0.13.1
+    - prometheus-client==0.7.1
+    - py==1.8.1
+    - pylint==2.4.4
+    - pyparsing==2.4.6
+    - pyrsistent==0.15.7
+    - pytest==5.4.1
+    - pytest-pylint==0.15.1
+    - python-slugify==4.0.0
+    - pytz==2019.3
+    - pyyaml==5.3
+    - requests==2.22.0
+    - scipy==1.4.1
+    - seaborn==0.10.0
+    - send2trash==1.5.0
+    - slurm2sql==0.9.0
+    - tangled-up-in-unicode==0.0.3
+    - terminado==0.8.3
+    - testpath==0.4.4
+    - text-unidecode==1.3
+    - tqdm==4.42.0
+    - urllib3==1.25.8
+    - visions==0.2.2
+    - webencodings==0.5.1
+    - widgetsnbextension==3.5.1
+    - wrapt==1.11.2
+prefix: /home/user/.conda/envs/slurm-ds
+

requirements.txt

+astroid==2.3.3
+astropy==4.0
+attr==0.3.1
+attrs==19.3.0
+backcall==0.1.0
+bleach==3.1.3
+certifi==2019.11.28
+chardet==3.0.4
+confuse==1.0.0
+cycler==0.10.0
+decorator==4.4.2
+defusedxml==0.6.0
+entrypoints==0.3
+htmlmin==0.1.12
+idna==2.8
+ipykernel==5.1.4
+ipython==7.13.0
+ipython-genutils==0.2.0
+ipywidgets==7.5.1
+isort==4.3.21
+jedi==0.16.0
+Jinja2==2.11.1
+joblib==0.14.1
+jsonschema==3.2.0
+jupyter-client==6.0.0
+jupyter-core==4.6.1
+kaggle==1.5.6
+kiwisolver==1.1.0
+lazy-object-proxy==1.4.3
+llvmlite==0.31.0
+MarkupSafe==1.1.1
+matplotlib==3.2.0
+mccabe==0.6.1
+missingno==0.4.2
+mistune==0.8.4
+more-itertools==8.2.0
+nbconvert==5.6.1
+nbformat==5.0.4
+networkx==2.4
+notebook==6.0.3
+numba==0.48.0
+numpy==1.18.2
+packaging==20.3
+pandas==0.25.3
+pandas-profiling==2.5.0
+pandocfilters==1.4.2
+parso==0.6.2
+pexpect==4.8.0
+phik==0.9.9
+pickleshare==0.7.5
+pluggy==0.13.1
+prometheus-client==0.7.1
+prompt-toolkit==3.0.3
+ptyprocess==0.6.0
+py==1.8.1
+Pygments==2.6.1
+pylint==2.4.4
+pyparsing==2.4.6
+pyrsistent==0.15.7
+pytest==5.4.1
+pytest-pylint==0.15.1
+python-dateutil==2.8.1
+python-slugify==4.0.0
+pytz==2019.3
+PyYAML==5.3
+pyzmq==18.1.1
+requests==2.22.0
+scipy==1.4.1
+seaborn==0.10.0
+Send2Trash==1.5.0
+six==1.14.0
+slurm2sql==0.9.0
+tangled-up-in-unicode==0.0.3
+terminado==0.8.3
+testpath==0.4.4
+text-unidecode==1.3
+tornado==6.0.4
+tqdm==4.42.0
+traitlets==4.3.3
+urllib3==1.25.8
+visions==0.2.2
+wcwidth==0.1.8
+webencodings==0.5.1
+widgetsnbextension==3.5.1
+wrapt==1.11.2

slurm-2sql.ipynb

+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import sqlite3\n",
+    "import slurm2sql\n",
+    "import pandas as pd"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0"
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "db = sqlite3.connect('test.db')\n",
+    "slurm2sql.slurm2sql(db, ['-S', '2020-03-18', '-a'])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# For example, you can then convert to a dataframe:\n",
+    "df1 = pd.read_sql('SELECT * FROM slurm', db)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>JobID</th>\n",
+       "      <th>ArrayJobID</th>\n",
+       "      <th>ArrayTaskID</th>\n",
+       "      <th>JobStep</th>\n",
+       "      <th>JobIDSlurm</th>\n",
+       "      <th>JobName</th>\n",
+       "      <th>User</th>\n",
+       "      <th>Group</th>\n",
+       "      <th>Account</th>\n",
+       "      <th>State</th>\n",
+       "      <th>...</th>\n",
+       "      <th>MaxDiskReadNode</th>\n",
+       "      <th>MaxDiskReadTask</th>\n",
+       "      <th>MaxDiskWrite</th>\n",
+       "      <th>MaxDiskWriteNode</th>\n",
+       "      <th>MaxDiskWriteTask</th>\n",
+       "      <th>ReqGPUS</th>\n",
+       "      <th>Comment</th>\n",
+       "      <th>GPUMem</th>\n",
+       "      <th>GPUEff</th>\n",
+       "      <th>NGPU</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>3319116</td>\n",
+       "      <td>3319116</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>None</td>\n",
+       "      <td>3319116_[43-45,47%5]</td>\n",
+       "      <td>1mUD1MPa</td>\n",
+       "      <td>user</td>\n",
+       "      <td>user</td>\n",
+       "      <td>user</td>\n",
+       "      <td>PENDING</td>\n",
+       "      <td>...</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>NaN</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>3927198</td>\n",
+       "      <td>3887451</td>\n",
+       "      <td>30.0</td>\n",
+       "      <td>None</td>\n",
+       "      <td>3887451_30</td>\n",
+       "      <td>100kCrC20MPa</td>\n",
+       "      <td>user</td>\n",
+       "      <td>user</td>\n",
+       "      <td>user</td>\n",
+       "      <td>COMPLETED</td>\n",
+       "      <td>...</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>NaN</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>3927198</td>\n",
+       "      <td>3887451</td>\n",
+       "      <td>30.0</td>\n",
+       "      <td>batch</td>\n",
+       "      <td>3887451_30.batch</td>\n",
+       "      <td>batch</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>user</td>\n",
+       "      <td>COMPLETED</td>\n",
+       "      <td>...</td>\n",
+       "      <td>c0088</td>\n",
+       "      <td>0</td>\n",
+       "      <td>1.222336e+10</td>\n",
+       "      <td>c0088</td>\n",
+       "      <td>0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>3927198</td>\n",
+       "      <td>3887451</td>\n",
+       "      <td>30.0</td>\n",
+       "      <td>extern</td>\n",
+       "      <td>3887451_30.extern</td>\n",
+       "      <td>extern</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>user</td>\n",
+       "      <td>COMPLETED</td>\n",
+       "      <td>...</td>\n",
+       "      <td>c0088</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.000000e+00</td>\n",
+       "      <td>c0088</td>\n",
+       "      <td>0</td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>3927199</td>\n",
+       "      <td>3887451</td>\n",
+       "      <td>31.0</td>\n",
+       "      <td>None</td>\n",
+       "      <td>3887451_31</td>\n",
+       "      <td>100kCrC20MPa</td>\n",
+       "      <td>user</td>\n",
+       "      <td>user</td>\n",
+       "      <td>user</td>\n",
+       "      <td>COMPLETED</td>\n",
+       "      <td>...</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>NaN</td>\n",
+       "      <td></td>\n",
+       "      <td></td>\n",
+       "      <td>NaN</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "      <td>None</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>5 rows × 63 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "     JobID  ArrayJobID  ArrayTaskID JobStep            JobIDSlurm  \\\n",
+       "0  3319116     3319116          NaN    None  3319116_[43-45,47%5]   \n",
+       "1  3927198     3887451         30.0    None            3887451_30   \n",
+       "2  3927198     3887451         30.0   batch      3887451_30.batch   \n",
+       "3  3927198     3887451         30.0  extern     3887451_30.extern   \n",
+       "4  3927199     3887451         31.0    None            3887451_31   \n",
+       "\n",
+       "        JobName      User     Group   Account      State  ...  \\\n",
+       "0      1mUD1MPa  user  user  user    PENDING  ...   \n",
+       "1  100kCrC20MPa  user  user  user  COMPLETED  ...   \n",
+       "2         batch                      user  COMPLETED  ...   \n",
+       "3        extern                      user  COMPLETED  ...   \n",
+       "4  100kCrC20MPa  user  user  user  COMPLETED  ...   \n",
+       "\n",
+       "   MaxDiskReadNode  MaxDiskReadTask  MaxDiskWrite  MaxDiskWriteNode  \\\n",
+       "0                                             NaN                     \n",
+       "1                                             NaN                     \n",
+       "2            c0088                0  1.222336e+10             c0088   \n",
+       "3            c0088                0  0.000000e+00             c0088   \n",
+       "4                                             NaN                     \n",
+       "\n",
+       "   MaxDiskWriteTask  ReqGPUS Comment GPUMem  GPUEff  NGPU  \n",
+       "0                        NaN    None   None    None  None  \n",
+       "1                        NaN    None   None    None  None  \n",
+       "2                 0      NaN    None   None    None  None  \n",
+       "3                 0      NaN    None   None    None  None  \n",
+       "4                        NaN    None   None    None  None  \n",
+       "\n",
+       "[5 rows x 63 columns]"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df1.head(5)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python [conda env:.conda-slurm-ds]",
+   "language": "python",
+   "name": "conda-env-.conda-slurm-ds-py"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.1"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}