Gitlab has been upgraded from version 14.2 to 14.4. Release notes can be found here (we run the Core release): https://about.gitlab.com/releases/2021/10/22/gitlab-14-4-released/

Commit 5d01b229 authored by William Stonewall Monroe's avatar William Stonewall Monroe
Browse files

Initial commit

parents
This diff is collapsed.
#!/bin/sh
directoryToUse="/data/user/$USER/"
sacct -P -u $USER --starttime=2019-01-01 --format user,start,jobid,jobname,state,partition,maxrss,reqmem,reqcpus,node,nnodes,elapsed >> "$directoryToUse"userusage.txt
{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"metadata": {
"ExecuteTime": {
"end_time": "2020-03-16T20:57:10.405006Z",
"start_time": "2020-03-16T20:56:55.837670Z"
}
},
"outputs": [],
"source": [
"import numpy as np\n",
"import pandas as pd\n",
"import pandas_profiling"
]
},
{
"cell_type": "code",
"execution_count": 2,
"metadata": {
"ExecuteTime": {
"end_time": "2020-03-16T20:57:11.865980Z",
"start_time": "2020-03-16T20:57:10.414986Z"
}
},
"outputs": [],
"source": [
"df = pd.read_csv('userusage.txt',delimiter='|')"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {
"ExecuteTime": {
"end_time": "2020-03-16T20:57:11.932878Z",
"start_time": "2020-03-16T20:57:11.905219Z"
}
},
"outputs": [
{
"data": {
"text/html": [
"<div>\n",
"<style scoped>\n",
" .dataframe tbody tr th:only-of-type {\n",
" vertical-align: middle;\n",
" }\n",
"\n",
" .dataframe tbody tr th {\n",
" vertical-align: top;\n",
" }\n",
"\n",
" .dataframe thead th {\n",
" text-align: right;\n",
" }\n",
"</style>\n",
"<table border=\"1\" class=\"dataframe\">\n",
" <thead>\n",
" <tr style=\"text-align: right;\">\n",
" <th></th>\n",
" <th>User</th>\n",
" <th>Start</th>\n",
" <th>JobID</th>\n",
" <th>JobName</th>\n",
" <th>State</th>\n",
" <th>Partition</th>\n",
" <th>MaxRSS</th>\n",
" <th>ReqMem</th>\n",
" <th>ReqCPUS</th>\n",
" <th>NodeList</th>\n",
" <th>NNodes</th>\n",
" <th>Elapsed</th>\n",
" </tr>\n",
" </thead>\n",
" <tbody>\n",
" <tr>\n",
" <th>0</th>\n",
" <td>user</td>\n",
" <td>2019-01-06T22:00:21</td>\n",
" <td>2040834</td>\n",
" <td>_interactive</td>\n",
" <td>COMPLETED</td>\n",
" <td>medium</td>\n",
" <td>NaN</td>\n",
" <td>10000Mc</td>\n",
" <td>1</td>\n",
" <td>c0088</td>\n",
" <td>1</td>\n",
" <td>16:04:23</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>NaN</td>\n",
" <td>2019-01-06T22:00:21</td>\n",
" <td>2040834.batch</td>\n",
" <td>batch</td>\n",
" <td>COMPLETED</td>\n",
" <td>NaN</td>\n",
" <td>1394528K</td>\n",
" <td>10000Mc</td>\n",
" <td>1</td>\n",
" <td>c0088</td>\n",
" <td>1</td>\n",
" <td>16:04:23</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>user</td>\n",
" <td>2019-01-07T16:15:21</td>\n",
" <td>2043373</td>\n",
" <td>Pipe_trim_galore</td>\n",
" <td>COMPLETED</td>\n",
" <td>medium</td>\n",
" <td>NaN</td>\n",
" <td>2000Mc</td>\n",
" <td>1</td>\n",
" <td>c0038</td>\n",
" <td>1</td>\n",
" <td>00:18:41</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>NaN</td>\n",
" <td>2019-01-07T16:15:21</td>\n",
" <td>2043373.batch</td>\n",
" <td>batch</td>\n",
" <td>COMPLETED</td>\n",
" <td>NaN</td>\n",
" <td>58592K</td>\n",
" <td>2000Mc</td>\n",
" <td>1</td>\n",
" <td>c0038</td>\n",
" <td>1</td>\n",
" <td>00:18:41</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>user</td>\n",
" <td>2019-01-07T16:15:21</td>\n",
" <td>2043374</td>\n",
" <td>Pipe_trim_galore</td>\n",
" <td>COMPLETED</td>\n",
" <td>medium</td>\n",
" <td>NaN</td>\n",
" <td>2000Mc</td>\n",
" <td>1</td>\n",
" <td>c0063</td>\n",
" <td>1</td>\n",
" <td>00:15:48</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" User Start JobID JobName State \\\n",
"0 user 2019-01-06T22:00:21 2040834 _interactive COMPLETED \n",
"1 NaN 2019-01-06T22:00:21 2040834.batch batch COMPLETED \n",
"2 user 2019-01-07T16:15:21 2043373 Pipe_trim_galore COMPLETED \n",
"3 NaN 2019-01-07T16:15:21 2043373.batch batch COMPLETED \n",
"4 user 2019-01-07T16:15:21 2043374 Pipe_trim_galore COMPLETED \n",
"\n",
" Partition MaxRSS ReqMem ReqCPUS NodeList NNodes Elapsed \n",
"0 medium NaN 10000Mc 1 c0088 1 16:04:23 \n",
"1 NaN 1394528K 10000Mc 1 c0088 1 16:04:23 \n",
"2 medium NaN 2000Mc 1 c0038 1 00:18:41 \n",
"3 NaN 58592K 2000Mc 1 c0038 1 00:18:41 \n",
"4 medium NaN 2000Mc 1 c0063 1 00:15:48 "
]
},
"execution_count": 3,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df.head()"
]
},
{
"cell_type": "code",
"execution_count": 4,
"metadata": {
"ExecuteTime": {
"end_time": "2020-03-16T20:57:14.154962Z",
"start_time": "2020-03-16T20:57:11.967976Z"
}
},
"outputs": [
{
"data": {
"text/plain": [
"array(['medium', nan, 'medium', ..., 'medium', nan, nan], dtype=object)"
]
},
"execution_count": 4,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"df[['jid','step']] = df.JobID.str.split(\".\",expand=True) \n",
"df.Partition.values"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {
"ExecuteTime": {
"start_time": "2020-03-16T20:56:57.392Z"
}
},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"/home/wsmonroe/.conda/envs/wsmplayground/lib/python3.6/site-packages/ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame\n",
"\n",
"See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
" after removing the cwd from sys.path.\n",
"/home/wsmonroe/.conda/envs/wsmplayground/lib/python3.6/site-packages/pandas/core/generic.py:7626: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame\n",
"\n",
"See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
" self._update_inplace(new_data)\n",
"/home/wsmonroe/.conda/envs/wsmplayground/lib/python3.6/site-packages/IPython/core/interactiveshell.py:2961: SettingWithCopyWarning: \n",
"A value is trying to be set on a copy of a slice from a DataFrame\n",
"\n",
"See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
" exec(code_obj, self.user_global_ns, self.user_ns)\n"
]
}
],
"source": [
"batchDF=df.dropna(subset=[\"MaxRSS\"])\n",
"userDF=df.dropna(subset=[\"User\"])\n",
"for jid in df.jid.unique():\n",
" userDF['MaxRSS'][userDF['jid'] == jid]=batchDF['MaxRSS'][batchDF['jid'] == jid]\n",
" \n",
" #print(userDF[userDF['jid'] == jid])\n",
" \n",
"userDF.head()"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python [conda env:wsmplayground]",
"language": "python",
"name": "conda-env-wsmplayground-py"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.6.8"
},
"toc": {
"base_numbering": 1,
"nav_menu": {},
"number_sections": true,
"sideBar": true,
"skip_h1_title": false,
"title_cell": "Table of Contents",
"title_sidebar": "Contents",
"toc_cell": false,
"toc_position": {},
"toc_section_display": true,
"toc_window_display": false
},
"varInspector": {
"cols": {
"lenName": 16,
"lenType": 16,
"lenVar": 40
},
"kernels_config": {
"python": {
"delete_cmd_postfix": "",
"delete_cmd_prefix": "del ",
"library": "var_list.py",
"varRefreshCmd": "print(var_dic_list())"
},
"r": {
"delete_cmd_postfix": ") ",
"delete_cmd_prefix": "rm(",
"library": "var_list.r",
"varRefreshCmd": "cat(var_dic_list()) "
}
},
"types_to_exclude": [
"module",
"function",
"builtin_function_or_method",
"instance",
"_Feature"
],
"window_display": false
}
},
"nbformat": 4,
"nbformat_minor": 2
}
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment