Add notebooks for 2023-12-04

.gitignore

+.venv
+yolo*
\ No newline at end of file

1_web_resources.ipynb

+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Download various data formats from the internet using the Python `requests` library and save them to disk.\n",
+    "\n",
+    "Data formats:\n",
+    "  - XML\n",
+    "  - CSV\n",
+    "  - Image"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import requests\n",
+    "from pathlib import Path\n",
+    "\n",
+    "data_folder = Path('data')\n",
+    "data_folder.mkdir(exist_ok=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# XML\n",
+    "# TEI-XML file from the Blotius edition (https://edition.onb.ac.at/blotius)\n",
+    "xml_data = requests.get('https://edition.onb.ac.at/blotius/o:blo.inventar-mz/datastreams/TEI_SOURCE/content')\n",
+    "\n",
+    "with open(data_folder / 'tei-xml_example.xml', 'wb') as file:\n",
+    "    file.write(xml_data.content)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# CSV\n",
+    "# Historic postcards metadata sheet\n",
+    "\n",
+    "csv_data = requests.get('https://labs.onb.ac.at/gitlab/labs-team/raw-metadata/-/raw/master/akon_postcards_public_domain.csv.bz2?ref_type=heads&inline=false')\n",
+    "\n",
+    "with open(data_folder / 'csv_example.csv.bz2', 'wb') as file:\n",
+    "    file.write(csv_data.content)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# Image\n",
+    "# Stock image from pexels.com\n",
+    "img_data = requests.get('https://images.pexels.com/photos/572056/pexels-photo-572056.jpeg?cs=srgb&dl=pexels-serpstat-572056.jpg&fm=jpg&w=5664&h=3778')\n",
+    "\n",
+    "with open(data_folder / 'image_example.jpg', 'wb') as file:\n",
+    "    file.write(img_data.content)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

2_xml_data.ipynb

+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Process the TEI-XML file using the `lxml` library with basic queries."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from lxml import etree as et\n",
+    "from pathlib import Path\n",
+    "\n",
+    "data_folder = Path('data')\n",
+    "output_folder = Path('output')\n",
+    "output_folder.mkdir(exist_ok=True)\n",
+    "\n",
+    "namespaces = {\n",
+    "    'tei': 'http://www.tei-c.org/ns/1.0'\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "blotius_xml = et.parse(data_folder / 'tei-xml_example.xml')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for rs in blotius_xml.xpath('.//tei:rs', namespaces=namespaces):\n",
+    "    # list all attributes of rs-elements\n",
+    "    if rs.attrib['type'] != 'place':\n",
+    "        print(rs)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# count the number of entries\n",
+    "entries = blotius_xml.xpath('.//tei:div[@type = \"entry\"]', namespaces=namespaces)\n",
+    "print(len(entries))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# manipulate all entries\n",
+    "for entry in entries:\n",
+    "    entry.attrib['subtype'] = 'catalogue'"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# save manipulated xml to disk\n",
+    "blotius_xml.write(output_folder / 'tei-xml_example.xml')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

3_csv_data.ipynb

+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Process csv data using *pandas* data frames"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "from pathlib import Path\n",
+    "from ultralytics import YOLO\n",
+    "\n",
+    "data_folder = Path('data')\n",
+    "output_folder = Path('output')\n",
+    "output_folder.mkdir(exist_ok=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "akon_data = pd.read_csv(data_folder / 'csv_example.csv.bz2', compression='bz2')\n",
+    "print(akon_data.head())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# save file without compression to disk, as Excel file and as csv dump\n",
+    "akon_data.to_excel(output_folder / 'csv_example.xlsx')\n",
+    "akon_data.to_csv(output_folder / 'csv_example.csv')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# get all black and white images\n",
+    "akon_bw = akon_data[akon_data['color'] == False]\n",
+    "akon_bw.to_excel(output_folder / 'csv_example_bw.xlsx')\n",
+    "print(akon_bw.head())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# get all images from DE before 1900\n",
+    "akon_de = akon_data[(akon_data['country_id'] == 'DE') & (akon_data['year'] < 1900)]\n",
+    "akon_de.to_excel(output_folder / 'csv_example_de.xlsx')\n",
+    "print(akon_de.head())"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

4_img_data.ipynb

+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Process images using Python Pillow library"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from PIL import Image, ImageDraw, ImageFilter\n",
+    "from ultralytics import YOLO\n",
+    "from pathlib import Path\n",
+    "\n",
+    "data_folder = Path('data')\n",
+    "output_folder = Path('output')\n",
+    "output_folder.mkdir(exist_ok=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "stock_image = Image.open(data_folder / 'image_example.jpg')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "print(stock_image.format, stock_image.size, stock_image.mode)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "stock_image_rect = ImageDraw.Draw(stock_image)\n",
+    "stock_image_rect.line((0, 0) + stock_image.size, fill=128)\n",
+    "stock_image_rect.rectangle([(300, 500), (1100, 900)], fill=128)\n",
+    "stock_image.save(output_folder / 'image_example_rect.png', 'PNG')\n",
+    "print(stock_image.getpixel((3000, 2500)))\n",
+    "# excursus digital image representation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "stock_image = Image.open(data_folder / 'image_example.jpg')\n",
+    "\n",
+    "stock_image_crop = stock_image.crop((200, 130, 500, 700))\n",
+    "stock_image_crop.save(output_folder / 'image_example_crop.png', 'PNG')\n",
+    "\n",
+    "stock_image_grey = stock_image.convert('L')\n",
+    "stock_image_grey.save(output_folder / 'image_example_grey.png', 'PNG')\n",
+    "\n",
+    "stock_image_edges = stock_image.filter(ImageFilter.FIND_EDGES)\n",
+    "stock_image_edges.save(output_folder / 'image_example_edges.png', 'PNG')\n",
+    "\n",
+    "stock_image_blur = stock_image.filter(ImageFilter.BLUR)\n",
+    "stock_image_blur.save(output_folder / 'image_example_blur.png', 'PNG')\n",
+    "\n",
+    "stock_image_contour = stock_image.filter(ImageFilter.CONTOUR)\n",
+    "stock_image_contour.save(output_folder / 'image_example_contour.png', 'PNG')\n",
+    "\n",
+    "stock_image_edge_enhance = stock_image.filter(ImageFilter.EDGE_ENHANCE)\n",
+    "stock_image_edge_enhance.save(output_folder / 'image_example_edge_enhance.png', 'PNG')\n",
+    "\n",
+    "stock_image_sharpen = stock_image.filter(ImageFilter.SHARPEN)\n",
+    "stock_image_sharpen.save(output_folder / 'image_example_sharpen.png', 'PNG')\n",
+    "\n",
+    "stock_image_emboss = stock_image.filter(ImageFilter.EMBOSS)\n",
+    "stock_image_emboss.save(output_folder / 'image_example_emboss.png', 'PNG')\n",
+    "\n",
+    "stock_image_smooth = stock_image.filter(ImageFilter.SMOOTH)\n",
+    "stock_image_smooth.save(output_folder / 'image_example_smooth.png', 'PNG')\n",
+    "\n",
+    "stock_image_detail = stock_image.filter(ImageFilter.DETAIL)\n",
+    "stock_image_detail.save(output_folder / 'image_example_detail.png', 'PNG')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Classify image using ultralytics"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# detect objects in image\n",
+    "model = YOLO('yolov8n.pt')  # load an official model\n",
+    "\n",
+    "results = model(data_folder / 'image_example.jpg')\n",
+    "for r in results:\n",
+    "    im_array = r.plot()\n",
+    "    im = Image.fromarray(im_array[..., ::-1])\n",
+    "    im.save(output_folder / 'image_example_ultralytics_detect.png')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# image classification\n",
+    "model = YOLO('yolov8n-cls.pt')  # load an official model\n",
+    "\n",
+    "results = model(data_folder / 'image_example.jpg')\n",
+    "for r in results:\n",
+    "    im_array = r.plot()\n",
+    "    im = Image.fromarray(im_array[..., ::-1])\n",
+    "    im.save(output_folder / 'image_example_ultralytics_predict.png')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# image segmentation\n",
+    "model = YOLO('yolov8n-seg.pt')  # load an official model\n",
+    "\n",
+    "results = model(data_folder / 'image_example.jpg')\n",
+    "for r in results:\n",
+    "    print(r)\n",
+    "    im_array = r.plot()\n",
+    "    im = Image.fromarray(im_array[..., ::-1])\n",
+    "    im.save(output_folder / 'image_example_ultralytics_segment.png')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# task\n",
+    "  - download an image\n",
+    "  - object detection\n",
+    "  - save detected objects as snippets"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.2"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

README.md

+# Python Introduction
+2023W 070128-1 Methodological course - Analyzing Central Europe's Intellectual Heritage
+
+## General Shortcuts
+
+`Ctrl + Shift + X`: extension manager
+
+`Ctrl + Shift + P`: command palette
+
+`Ctrl + Shift + G`: source control view
+
+## Repository Setup
+1. fork this repository in GitLab to have a private copy
+2. open VS Code
+3. clone git repository via command palette: `Git: Clone`
+  - use `https` and not `ssh` with your username and password
+  - if asked, open cloned repository in a new window
+4. download an image from the internet and add it to this readme
+5. make a git commit via the source control view and push the changes to GitLab
+
+## Python Setup
+1. create virtual environment `.venv` via command palette: `Python: Create environment`
+2. install project requirements
+  - open a new terminal via menu: (`Terminal > New Terminal`)
+  - verify you are using the virtual environment
+  - install dependencies via python packet manager `pip install -r requirements.txt`
+  - list the installed packages via python packet manager `pip freeze`
+  - close the terminal
+3. open the first Jupyter Notebook and install package if needed
+
+## Lesson 1 - Web Resources
+This notebooks gives an introduction how to us data sources from the internet.

requirements.txt

+lxml==4.9.3
+openpyxl==3.1.2
+pandas==2.1.3
+Pillow==10.1.0
+requests==2.31.0
+ultralytics==8.0.222
\ No newline at end of file