diff --git a/Python/QA-003-Conda-Environment.ipynb b/Python/QA-003-Conda-Environment.ipynb
index 7786f777968b6f978140cb215f9ed89cf72b9194..cc8118bf5dfbf6a202ef42fda552a6400323f04d 100644
--- a/Python/QA-003-Conda-Environment.ipynb
+++ b/Python/QA-003-Conda-Environment.ipynb
@@ -428,7 +428,7 @@
],
"metadata": {
"kernelspec": {
- "display_name": "Anaconda3 (2020.07)",
+ "display_name": "Miniconda3 (4.10.3) - 3.9.5",
"language": "python",
"name": "python3"
},
@@ -442,7 +442,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.8.3"
+ "version": "3.9.5"
}
},
"nbformat": 4,
diff --git a/Python/QA-009-Memory-Profiling.ipynb b/Python/QA-009-Memory-Profiling.ipynb
new file mode 100644
index 0000000000000000000000000000000000000000..70efc514d79c522e58c452e94e1962183cd1ae0c
--- /dev/null
+++ b/Python/QA-009-Memory-Profiling.ipynb
@@ -0,0 +1,350 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "7e3efaba-dc8d-4f02-b599-1d8e8d8a12e5",
+ "metadata": {},
+ "source": [
+ "# Memory Profiling in Python\n",
+ "\n",
+ "It is quite important to know what is going wrong in your code, to perform better.\n",
+ "One tool to help you do what you need to do is to use a memory profiler.\n",
+ "\n",
+ "- [PyPI - Memory Profiler](https://pypi.org/project/memory-profiler/)\n",
+ "\n",
+ "for execution speed use:\n",
+ "\n",
+ "- [PyPI - Line Profiler](https://github.com/pyutils/line_profiler)\n",
+ "- [CProfile](https://docs.python.org/3.8/library/profile.html)\n",
+ "\n",
+ "Install memory profile via pip:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "id": "2132773b-1236-482b-8d2d-b1bd5e1a70d5",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!pip install --user --quiet memory_profiler"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "58a74d61-e7ab-4216-ba35-6de3f5aced5f",
+ "metadata": {},
+ "source": [
+ "After installing you might need to restart your kernel!\n",
+ "\n",
+ "\n",
+ "Write an example with a function and add the `@profile` decorator to indicate which function shall be profiled."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "id": "e3df4ef0-c97a-49f7-b1a3-36f08d88f7be",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Overwriting example.py\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%writefile example.py\n",
+ "\n",
+ "@profile\n",
+ "def my_func():\n",
+ " a = [1] * (10 ** 6)\n",
+ " b = [2] * (2 * 10 ** 7)\n",
+ " del b\n",
+ " return a\n",
+ "\n",
+ "if __name__ == '__main__':\n",
+ " my_func()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "id": "1de3b0c2-27a1-4182-bbdc-1b67764e4d62",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Filename: example.py\n",
+ "\n",
+ "Line # Mem usage Increment Occurences Line Contents\n",
+ "============================================================\n",
+ " 2 39.406 MiB 39.406 MiB 1 @profile\n",
+ " 3 def my_func():\n",
+ " 4 47.141 MiB 7.734 MiB 1 a = [1] * (10 ** 6)\n",
+ " 5 199.766 MiB 152.625 MiB 1 b = [2] * (2 * 10 ** 7)\n",
+ " 6 47.254 MiB -152.512 MiB 1 del b\n",
+ " 7 47.254 MiB 0.000 MiB 1 return a\n",
+ "\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "# Execute the module with the memory_profiler being loaded:\n",
+ "!python3 -m memory_profiler example.py"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "5bdad5f1-fb41-45b1-8cb0-e0c93cf5834a",
+ "metadata": {},
+ "source": [
+ "Of course you can also add this directly into the file:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "id": "6dda1f01-ae3b-4ccd-af78-6c5c3c227ad6",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Overwriting example.py\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%writefile example.py\n",
+ "from memory_profiler import profile\n",
+ "\n",
+ "@profile\n",
+ "def my_func():\n",
+ " a = [1] * (10 ** 6)\n",
+ " b = [2] * (2 * 10 ** 7)\n",
+ " del b\n",
+ " return a\n",
+ "\n",
+ "if __name__ == '__main__':\n",
+ " my_func()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "id": "78036b7a-996e-43e5-88ba-c1f9120fc05b",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Filename: /home/spack/computer-resources/Python/example.py\n",
+ "\n",
+ "Line # Mem usage Increment Occurences Line Contents\n",
+ "============================================================\n",
+ " 3 39.4 MiB 39.4 MiB 1 @profile\n",
+ " 4 def my_func():\n",
+ " 5 47.1 MiB 7.7 MiB 1 a = [1] * (10 ** 6)\n",
+ " 6 199.7 MiB 152.6 MiB 1 b = [2] * (2 * 10 ** 7)\n",
+ " 7 47.2 MiB -152.5 MiB 1 del b\n",
+ " 8 47.2 MiB 0.0 MiB 1 return a\n",
+ "\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "!python3 example.py"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "5deb5183-9549-4ad3-90c0-2c51c8de783d",
+ "metadata": {},
+ "source": [
+ "It can be loaded into an ipython session with\n",
+ "`%load_ext memory_profiler`\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "id": "ef87b3f1-1d2b-4f3d-a6bb-5458a3cbd4ce",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%load_ext memory_profiler"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "id": "37eecb4f-b8b4-44c2-be4c-8845d3dc00bb",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Overwriting example.py\n"
+ ]
+ }
+ ],
+ "source": [
+ "%%writefile example.py\n",
+ "def my_func():\n",
+ " a = [1] * (10 ** 6)\n",
+ " b = [2] * (2 * 10 ** 7)\n",
+ " del b\n",
+ " return a\n",
+ "\n",
+ "if __name__ == '__main__':\n",
+ " my_func()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "id": "e6514424-09ff-494a-970d-9f32764ebd93",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from example import my_func"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "id": "6b1ed020-633e-4501-a3c3-a984ecab8bc4",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n"
+ ]
+ },
+ {
+ "data": {
+ "text/plain": [
+ "Filename: /home/spack/computer-resources/Python/example.py\n",
+ "\n",
+ "Line # Mem usage Increment Occurences Line Contents\n",
+ "============================================================\n",
+ " 1 52.2 MiB 52.2 MiB 1 def my_func():\n",
+ " 2 59.8 MiB 7.6 MiB 1 a = [1] * (10 ** 6)\n",
+ " 3 212.4 MiB 152.6 MiB 1 b = [2] * (2 * 10 ** 7)\n",
+ " 4 59.9 MiB -152.6 MiB 1 del b\n",
+ " 5 59.9 MiB 0.0 MiB 1 return a"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "%mprun -f my_func my_func()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "46720507-a09b-4efd-91be-4bf2bc802596",
+ "metadata": {},
+ "source": [
+ "## Line profiler\n",
+ "\n",
+ "Line profiler works similar."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "id": "204afab1-04b8-4c48-b343-becd070371a7",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "!pip install --user --quiet line_profiler"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "id": "96c33187-40f9-4885-af21-2cc2b6bd4035",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "%load_ext line_profiler"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "id": "d407f5d4-3aaa-4660-a9e5-432613dd5259",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "Timer unit: 1e-06 s\n",
+ "\n",
+ "Total time: 0.127 s\n",
+ "File: /home/spack/computer-resources/Python/example.py\n",
+ "Function: my_func at line 1\n",
+ "\n",
+ "Line # Hits Time Per Hit % Time Line Contents\n",
+ "==============================================================\n",
+ " 1 def my_func():\n",
+ " 2 1 4842.0 4842.0 3.8 a = [1] * (10 ** 6)\n",
+ " 3 1 78616.0 78616.0 61.9 b = [2] * (2 * 10 ** 7)\n",
+ " 4 1 43539.0 43539.0 34.3 del b\n",
+ " 5 1 3.0 3.0 0.0 return a"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "%lprun -f my_func my_func()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "b242ab95-7c07-4a98-bad4-1cf8693cdb58",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Miniconda3 (4.10.3) - 3.9.5",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.5"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
diff --git a/Python/README.md b/Python/README.md
index b71225cbfd091d7b9ec4ec3aae04cda7bbc10275..aa23581e4672deadc4ddb122a078b4b32b2722fe 100644
--- a/Python/README.md
+++ b/Python/README.md
@@ -60,3 +60,7 @@ Ever needed to backup a conda environment or copy from a colleague or designing
You just need to add a configuration option and then you can open the Dashboard in any Jupyterhub. Works on SRVX1 and JET.
+## Q: How to profile memory and exeution time of functions?
+[Profile](/QA-009-Memory-Profiling.ipynb)
+
+If you need to get a better understanding of you functions memory and execution time, try these profiling options.
\ No newline at end of file
diff --git a/Python/Your-First-Notebook-onJet_v2.ipynb b/Python/Your-First-Notebook-onJet_v2.ipynb
index 582ffc4cc376bff29d21b5a568abb96f8ce7dddf..5d3796666b3481e0df0f23bda55924264eaee227 100644
--- a/Python/Your-First-Notebook-onJet_v2.ipynb
+++ b/Python/Your-First-Notebook-onJet_v2.ipynb
@@ -350,9 +350,9 @@
" - [x] NetCDF (Parallel (NC3), NC4->HDF5)\n",
" - [x] Eccodes (2.18)\n",
" - [x] CDO\n",
- " - [ ] Intel compiled Libraries (License) \n",
- " - [ ] Emoslib \n",
- " - [ ] Ecaccess\n",
+ " - [x] Intel compiled Libraries (License) \n",
+ " - [x] Emoslib \n",
+ " - [x] Ecaccess\n",
" - [ ] RTTOV"
]
},
@@ -404,8 +404,10 @@
"source": [
"import os\n",
"import sys\n",
+ "import site\n",
"# Need to update search Path / only necessary here for the first time\n",
- "sys.path.append(os.path.expandvars(\"$HOME/.local/lib/python3.8/site-packages\"))"
+ "# sys.path.append(os.path.expandvars(\"$HOME/.local/lib/python3.8/site-packages\"))\n",
+ "sys.path.append(site.getusersitepackages())"
]
},
{
@@ -563,6 +565,29 @@
"!conda create -y -n myenv python=3.7 numpy ipykernel"
]
},
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Activating a conda environment\n",
+ "\n",
+ "It is not recommended to use `conda init bash`, because it will write that specific version into your `.bashrc` file and will result in errors when you move to another module or newer version.\n",
+ "\n",
+ "Hint: Checkout the guides on [Gitlab](https://gitlab.phaidra.org/imgw/computer-resources/-/blob/master/Python/QA-003-Conda-Environment.ipynb#Activate-your-conda-environment)\n",
+ "\n",
+ "### Load from the current conda module the BASH functionality (as in activate)\n",
+ "\n",
+ "```sh\n",
+ "source $(which conda)/../etc/profile.d/conda.sh\n",
+ "# now activating the environment works\n",
+ "conda activate myenv\n",
+ "```\n",
+ "\n",
+ "### Activate your kernel in your notebooks\n",
+ "\n",
+ "Please run the follwoing command and install the ipykernel, if it is not yet installed:"
+ ]
+ },
{
"cell_type": "code",
"execution_count": 17,
@@ -693,7 +718,7 @@
],
"metadata": {
"kernelspec": {
- "display_name": "Miniconda3 (4.8.2)",
+ "display_name": "Miniconda3 (4.10.3) - 3.9.5",
"language": "python",
"name": "python3"
},
@@ -707,7 +732,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
- "version": "3.8.1"
+ "version": "3.9.5"
}
},
"nbformat": 4,