Skip to content
Snippets Groups Projects
Commit b5a66e4c authored by Martin Weise's avatar Martin Weise
Browse files

updated table endpint and feature extract commentary

parent 75911770
No related branches found
No related tags found
2 merge requests!81New stable release,!80Multiple features connected with user management and ownership of databases
This commit is part of merge request !81. Comments created here will be created in the context of that merge request.
...@@ -259,7 +259,7 @@ class TableDataEndpointApi(object): ...@@ -259,7 +259,7 @@ class TableDataEndpointApi(object):
auth_settings = [] # noqa: E501 auth_settings = [] # noqa: E501
return self.api_client.call_api( return self.api_client.call_api(
'/api/container/{id}/database/{databaseId}/table/{tableId}/data', 'HEAD', '/api/container/{id}/database/{databaseId}/table/{tableId}/data', 'GET',
path_params, path_params,
query_params, query_params,
header_params, header_params,
...@@ -380,7 +380,7 @@ class TableDataEndpointApi(object): ...@@ -380,7 +380,7 @@ class TableDataEndpointApi(object):
auth_settings = [] # noqa: E501 auth_settings = [] # noqa: E501
return self.api_client.call_api( return self.api_client.call_api(
'/api/container/{id}/database/{databaseId}/table/{tableId}/data', 'GET', '/api/container/{id}/database/{databaseId}/table/{tableId}/data', 'HEAD',
path_params, path_params,
query_params, query_params,
header_params, header_params,
......
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
# Deposit # Deposit
Please create an account at [http://localhost:3000/register](http://localhost:3000/register) with `user:user` before executing.
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
import os.path import os.path
from os import listdir from os import listdir
from os.path import isfile, join from os.path import isfile, join
from api_document.api.document_endpoint_api import DocumentEndpointApi from api_document.api.document_endpoint_api import DocumentEndpointApi
from api_document.api.file_endpoint_api import FileEndpointApi from api_document.api.file_endpoint_api import FileEndpointApi
from api_authentication.api.authentication_endpoint_api import AuthenticationEndpointApi from api_authentication.api.authentication_endpoint_api import AuthenticationEndpointApi
authentication = AuthenticationEndpointApi() authentication = AuthenticationEndpointApi()
document = DocumentEndpointApi() document = DocumentEndpointApi()
file = FileEndpointApi() file = FileEndpointApi()
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
1. Obtain an authentication token 1. Obtain an authentication token
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
response = authentication.authenticate_user1({ response = authentication.authenticate_user1({
"username": "user", "username": "user",
"password": "user" "password": "user"
}) })
document.api_client.default_headers = {"Authorization": "Bearer " + response.token} document.api_client.default_headers = {"Authorization": "Bearer " + response.token}
file.api_client.default_headers = {"Authorization": "Bearer " + response.token} file.api_client.default_headers = {"Authorization": "Bearer " + response.token}
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
2. Create a metadata draft 2. Create a metadata draft
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
response = document.create({ response = document.create({
"access": { "access": {
"record": "public", "record": "public",
"files": "public" "files": "public"
}, },
"files": { "files": {
"enabled": True "enabled": True
}, },
"metadata": { "metadata": {
"creators": [ "creators": [
{ {
"affiliations": [ "affiliations": [
{ {
"name": "TU Wien" "name": "TU Wien"
} }
], ],
"person_or_org": { "person_or_org": {
"type": "personal", "type": "personal",
"name": "M., Weise", "name": "M., Weise",
"identifiers": [ "identifiers": [
{ {
"scheme": "orcid", "scheme": "orcid",
"identifier": "0000-0003-4216-302X" "identifier": "0000-0003-4216-302X"
} }
], ],
"given_name": "Martin", "given_name": "Martin",
"family_name": "Weise" "family_name": "Weise"
} }
} }
], ],
"title": "Jupyter Notebook Test", "title": "Jupyter Notebook Test",
"resource_type": { "resource_type": {
"id": "other" "id": "other"
}, },
"publication_date": "2022-06-28" "publication_date": "2022-06-28"
} }
}) })
document_id = response.id document_id = response.id
print(response) print(response)
``` ```
%% Output %% Output
{'access': {'embargo': {'active': False, 'reason': None, 'until': None}, {'access': {'embargo': {'active': False, 'reason': None, 'until': None},
'files': 'public', 'files': 'public',
'record': 'public'}, 'record': 'public'},
'created': datetime.datetime(2022, 6, 28, 19, 47, 36, 289193, tzinfo=tzutc()), 'created': datetime.datetime(2022, 6, 29, 10, 23, 1, 849279, tzinfo=tzutc()),
'expires_at': datetime.datetime(2022, 6, 28, 19, 47, 36, 289218), 'expires_at': datetime.datetime(2022, 6, 29, 10, 23, 1, 849305),
'files': {'default_preview': None, 'enabled': True}, 'files': {'default_preview': None, 'enabled': True},
'id': 'a13x6-2wz51', 'id': 'w37m8-dx896',
'is_published': False, 'is_published': False,
'metadata': {'creators': [{'affiliations': [{'id': None, 'name': 'TU Wien'}], 'metadata': {'creators': [{'affiliations': [{'id': None, 'name': 'TU Wien'}],
'person_or_org': {'family_name': 'Weise', 'person_or_org': {'family_name': 'Weise',
'given_name': 'Martin', 'given_name': 'Martin',
'identifiers': [{'identifier': '0000-0003-4216-302X', 'identifiers': [{'identifier': '0000-0003-4216-302X',
'scheme': 'orcid'}], 'scheme': 'orcid'}],
'name': 'Weise, Martin', 'name': 'Weise, Martin',
'type': 'personal'}, 'type': 'personal'},
'role': None}], 'role': None}],
'publication_date': datetime.datetime(2022, 6, 28, 0, 0), 'publication_date': datetime.datetime(2022, 6, 28, 0, 0),
'resource_type': {'id': 'other'}, 'resource_type': {'id': 'other'},
'title': 'Jupyter Notebook Test'}, 'title': 'Jupyter Notebook Test'},
'pids': {'doi': None}, 'pids': {'doi': None},
'revision_id': 4, 'revision_id': 4,
'updated': datetime.datetime(2022, 6, 28, 19, 47, 36, 304744, tzinfo=tzutc()), 'updated': datetime.datetime(2022, 6, 29, 10, 23, 1, 866243, tzinfo=tzutc()),
'versions': {'index': 1, 'is_latest': False, 'is_latest_draft': True}} 'versions': {'index': 1, 'is_latest': False, 'is_latest_draft': True}}
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
3. Upload the files 3. Upload the files
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
files = [f for f in listdir("./audio") if isfile(join("./audio", f))] files = [f for f in listdir("./audio") if isfile(join("./audio", f))]
for f in files: for f in files:
print("... upload file", "/tmp/" + f) print("... upload file", "/tmp/" + f)
response = file.upload_file({ response = file.upload_file({
"location": os.path.curdir + "/tmp/" + f "location": os.path.curdir + "/tmp/" + f
}, document_id) }, document_id)
``` ```
%% Output %% Output
... upload file /tmp/colive.0044_20200518133554_1_m4a_1.wav ... upload file /tmp/colive.0044_20200518133554_1_m4a_1.wav
... upload file /tmp/colive.0044_20200518133554_2_m4a_1.wav ... upload file /tmp/colive.0044_20200518133554_2_m4a_1.wav
... upload file /tmp/colive.0066_20200611134530_1_m4a_0.wav ... upload file /tmp/colive.0066_20200611134530_1_m4a_0.wav
... upload file /tmp/colive.0066_20200611134530_2_m4a_0.wav ... upload file /tmp/colive.0066_20200611134530_2_m4a_0.wav
... upload file /tmp/colive.0066_20200612072315_1_m4a_0.wav ... upload file /tmp/colive.0066_20200612072315_1_m4a_0.wav
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
4. Publish the record 4. Publish the record
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
response = document.publish(document_id) response = document.publish(document_id)
print(response) print(response)
``` ```
%% Output %% Output
{'access': {'embargo': {'active': False, 'reason': None, 'until': None}, {'access': {'embargo': {'active': False, 'reason': None, 'until': None},
'files': 'public', 'files': 'public',
'record': 'public'}, 'record': 'public'},
'created': datetime.datetime(2022, 6, 28, 19, 47, 42, 416073, tzinfo=tzutc()), 'created': datetime.datetime(2022, 6, 29, 10, 23, 3, 488874, tzinfo=tzutc()),
'expires_at': None, 'expires_at': None,
'files': {'default_preview': None, 'enabled': True}, 'files': {'default_preview': None, 'enabled': True},
'id': 'a13x6-2wz51', 'id': 'w37m8-dx896',
'is_published': True, 'is_published': True,
'metadata': {'creators': [{'affiliations': [{'id': None, 'name': 'TU Wien'}], 'metadata': {'creators': [{'affiliations': [{'id': None, 'name': 'TU Wien'}],
'person_or_org': {'family_name': 'Weise', 'person_or_org': {'family_name': 'Weise',
'given_name': 'Martin', 'given_name': 'Martin',
'identifiers': [{'identifier': '0000-0003-4216-302X', 'identifiers': [{'identifier': '0000-0003-4216-302X',
'scheme': 'orcid'}], 'scheme': 'orcid'}],
'name': 'Weise, Martin', 'name': 'Weise, Martin',
'type': 'personal'}, 'type': 'personal'},
'role': None}], 'role': None}],
'publication_date': datetime.datetime(2022, 6, 28, 0, 0), 'publication_date': datetime.datetime(2022, 6, 28, 0, 0),
'resource_type': {'id': 'other'}, 'resource_type': {'id': 'other'},
'title': 'Jupyter Notebook Test'}, 'title': 'Jupyter Notebook Test'},
'pids': {'doi': {'client': 'datacite', 'pids': {'doi': {'client': 'datacite',
'identifier': '10.70124/a13x6-2wz51', 'identifier': '10.70124/w37m8-dx896',
'provider': 'datacite'}}, 'provider': 'datacite'}},
'revision_id': 3, 'revision_id': 3,
'updated': datetime.datetime(2022, 6, 28, 19, 47, 42, 490667, tzinfo=tzutc()), 'updated': datetime.datetime(2022, 6, 29, 10, 23, 3, 555616, tzinfo=tzutc()),
'versions': {'index': 1, 'is_latest': True, 'is_latest_draft': True}} 'versions': {'index': 1, 'is_latest': True, 'is_latest_draft': True}}
......
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
# Feature Extraction & Deposit # Feature Extraction & Deposit
In this notebook we define an example of creating a database from a .csv and perform feature extraction of audio files. The APIs are auto-generated from the Swagger Endpoint documentations using [`generate.sh`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-docs/-/blob/master/swagger/generate.sh). Steps we perform: In this notebook we define an example of creating a database from a .csv and perform feature extraction of audio files. The APIs are auto-generated from the Swagger Endpoint documentations using [`generate.sh`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-docs/-/blob/master/swagger/generate.sh). Steps we perform:
1. Download a music file from a public repository 1. Download a music file from a public repository
2. Perform feature extraction 2. Perform feature extraction
3. Obtain an authentication token 3. Obtain an authentication token
4. Create a mariadb container 4. Create a mariadb container
5. Start the mariadb container 5. Start the mariadb container
6. Create a database within the mariadb container 6. Create a database within the mariadb container
7. Import the feature .csv (manually) 7. Import the feature .csv (manually)
Please create an account at [http://localhost:3000/register](http://localhost:3000/register) with `user:user` before executing.
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
import os.path import os.path
import uuid import uuid
import time import time
import re import re
import csv import csv
import requests as rq import requests as rq
from api_authentication.api.authentication_endpoint_api import AuthenticationEndpointApi from api_authentication.api.authentication_endpoint_api import AuthenticationEndpointApi
from api_authentication.api.user_endpoint_api import UserEndpointApi from api_authentication.api.user_endpoint_api import UserEndpointApi
from api_container.api.container_endpoint_api import ContainerEndpointApi from api_container.api.container_endpoint_api import ContainerEndpointApi
from api_database.api.container_database_endpoint_api import ContainerDatabaseEndpointApi from api_database.api.container_database_endpoint_api import ContainerDatabaseEndpointApi
from api_table.api.table_endpoint_api import TableEndpointApi from api_table.api.table_endpoint_api import TableEndpointApi
authentication = AuthenticationEndpointApi() authentication = AuthenticationEndpointApi()
user = UserEndpointApi() user = UserEndpointApi()
container = ContainerEndpointApi() container = ContainerEndpointApi()
database = ContainerDatabaseEndpointApi() database = ContainerDatabaseEndpointApi()
table = TableEndpointApi() table = TableEndpointApi()
doi = "10.5281/zenodo.5649276" doi = "10.5281/zenodo.5649276"
email = "some@example.com" email = "some@example.com"
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
8. Download wav 9. Download wav
Resolve the DOI to URI Resolve the DOI to URI
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
response = rq.get("https://doi.org/" + doi) response = rq.get("https://doi.org/" + doi)
id = re.findall("/([a-z0-9-]+)$", response.url)[0] id = re.findall("/([a-z0-9-]+)$", response.url)[0]
host = re.findall("^https?:\/\/([a-z0-9]+\.[a-z]+)", response.url)[0] host = re.findall("^https?:\/\/([a-z0-9]+\.[a-z]+)", response.url)[0]
print("Resolved DOI to", host, "and record id", id) print("Resolved DOI to", host, "and record id", id)
``` ```
%% Output %% Output
Resolved DOI to zenodo.org and record id 5649276 Resolved DOI to zenodo.org and record id 5649276
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
2. Perform feature extraction 2. Perform feature extraction
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
response = rq.get("https://" + host + "/api/records/" + id) response = rq.get("https://" + host + "/api/records/" + id)
record = response.json() record = response.json()
i = 0
with open(os.path.expanduser("~/features.csv"), "w") as f: with open(os.path.expanduser("~/features.csv"), "w") as f:
writer = csv.writer(f) writer = csv.writer(f)
writer.writerow(["key", "size", "link"]) writer.writerow(["key", "size", "link"])
for file in record["files"]: for file in record["files"]:
rq.get(file["links"]["self"]) rq.get(file["links"]["self"])
print("... feature extract from", file["links"]["self"]) print("... feature extract from", file["links"]["self"])
writer.writerow([file["key"], file["size"], file["links"]["self"]]) writer.writerow([file["key"], file["size"], file["links"]["self"]])
i += 1
if i > 5:
break
print("Generated a feature .csv in your home directory") print("Generated a feature .csv in your home directory")
``` ```
%% Output %% Output
... feature extract from https://zenodo.org/api/files/22d69a63-2aff-47ae-b818-be78a23e9889/colive.0044_20200518133554_1_m4a_1.wav ... feature extract from https://zenodo.org/api/files/22d69a63-2aff-47ae-b818-be78a23e9889/colive.0044_20200518133554_1_m4a_1.wav
... feature extract from https://zenodo.org/api/files/22d69a63-2aff-47ae-b818-be78a23e9889/colive.0044_20200518133554_2_m4a_1.wav ... feature extract from https://zenodo.org/api/files/22d69a63-2aff-47ae-b818-be78a23e9889/colive.0044_20200518133554_2_m4a_1.wav
... feature extract from https://zenodo.org/api/files/22d69a63-2aff-47ae-b818-be78a23e9889/colive.0066_20200611134530_1_m4a_0.wav ... feature extract from https://zenodo.org/api/files/22d69a63-2aff-47ae-b818-be78a23e9889/colive.0066_20200611134530_1_m4a_0.wav
... feature extract from https://zenodo.org/api/files/22d69a63-2aff-47ae-b818-be78a23e9889/colive.0066_20200611134530_2_m4a_0.wav ... feature extract from https://zenodo.org/api/files/22d69a63-2aff-47ae-b818-be78a23e9889/colive.0066_20200611134530_2_m4a_0.wav
... feature extract from https://zenodo.org/api/files/22d69a63-2aff-47ae-b818-be78a23e9889/colive.0066_20200612072315_1_m4a_0.wav ... feature extract from https://zenodo.org/api/files/22d69a63-2aff-47ae-b818-be78a23e9889/colive.0066_20200612072315_1_m4a_0.wav
... feature extract from https://zenodo.org/api/files/22d69a63-2aff-47ae-b818-be78a23e9889/colive.0066_20200612072315_2_m4a_0.wav ... feature extract from https://zenodo.org/api/files/22d69a63-2aff-47ae-b818-be78a23e9889/colive.0066_20200612072315_2_m4a_0.wav
... feature extract from https://zenodo.org/api/files/22d69a63-2aff-47ae-b818-be78a23e9889/colive.0066_20200613082517_1_m4a_0.wav
... feature extract from https://zenodo.org/api/files/22d69a63-2aff-47ae-b818-be78a23e9889/colive.0066_20200613082517_2_m4a_0.wav
... feature extract from https://zenodo.org/api/files/22d69a63-2aff-47ae-b818-be78a23e9889/colive.0066_20200614080017_1_m4a_0.wav
... feature extract from https://zenodo.org/api/files/22d69a63-2aff-47ae-b818-be78a23e9889/colive.0066_20200614080017_2_m4a_0.wav
... feature extract from https://zenodo.org/api/files/22d69a63-2aff-47ae-b818-be78a23e9889/colive.0066_20200615070238_1_m4a_0.wav
Generated a feature .csv in your home directory Generated a feature .csv in your home directory
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
3. Obtain an authentication token 3. Obtain an authentication token
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
response = authentication.authenticate_user1({ response = authentication.authenticate_user1({
"username": "user", "username": "user",
"password": "user" "password": "user"
}) })
container.api_client.default_headers = {"Authorization": "Bearer " + response.token} container.api_client.default_headers = {"Authorization": "Bearer " + response.token}
database.api_client.default_headers = {"Authorization": "Bearer " + response.token} database.api_client.default_headers = {"Authorization": "Bearer " + response.token}
table.api_client.default_headers = {"Authorization": "Bearer " + response.token} table.api_client.default_headers = {"Authorization": "Bearer " + response.token}
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
4. Create a mariadb container 4. Create a mariadb container
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
response = container.create1({ response = container.create1({
"name": "MIR " + str(uuid.uuid1()), "name": "MIR " + str(uuid.uuid1()),
"repository": "mariadb", "repository": "mariadb",
"tag": "10.5" "tag": "10.5"
}) })
container_id = response.id container_id = response.id
print(response) print(response)
``` ```
%% Output %% Output
{'hash': 'f5a649a71aae3748e62228721c44627ffc866f665d677bb890c37b9111590ffa', {'hash': 'a431232a07efc4027cf71beaf38e7465984985ded1d4a67a94d092b49bd4d65e',
'id': 2, 'id': 1,
'internal_name': 'fda-userdb-mir-1010b964-f6fa-11ec-9f77-64bc58900b78', 'internal_name': 'fda-userdb-mir-529d42b0-f796-11ec-ad95-64bc58900b78',
'is_public': None, 'is_public': None,
'name': 'MIR 1010b964-f6fa-11ec-9f77-64bc58900b78'} 'name': 'MIR 529d42b0-f796-11ec-ad95-64bc58900b78'}
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
5. Start the mariadb container 5. Start the mariadb container
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
response = container.modify({ response = container.modify({
"action": "START" "action": "START"
}, container_id) }, container_id)
time.sleep(5) time.sleep(5)
print(response) print(response)
``` ```
%% Output %% Output
{'hash': 'f5a649a71aae3748e62228721c44627ffc866f665d677bb890c37b9111590ffa', {'hash': 'a431232a07efc4027cf71beaf38e7465984985ded1d4a67a94d092b49bd4d65e',
'id': 2, 'id': 1,
'internal_name': 'fda-userdb-mir-1010b964-f6fa-11ec-9f77-64bc58900b78', 'internal_name': 'fda-userdb-mir-529d42b0-f796-11ec-ad95-64bc58900b78',
'is_public': None, 'is_public': None,
'name': 'MIR 1010b964-f6fa-11ec-9f77-64bc58900b78'} 'name': 'MIR 529d42b0-f796-11ec-ad95-64bc58900b78'}
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
6. Create a database within the mariadb container 6. Create a database within the mariadb container
%% Cell type:code id: tags: %% Cell type:code id: tags:
``` python ``` python
response = database.create({ response = database.create({
"name": "MIR " + str(uuid.uuid1()), "name": "MIR " + str(uuid.uuid1()),
"description": "Music Information Retrieval", "description": "Music Information Retrieval",
"is_public": True "is_public": True
}, container_id) }, container_id)
database_id = response.id database_id = response.id
``` ```
%% Cell type:markdown id: tags: %% Cell type:markdown id: tags:
7. Import the feature .csv 7. Import the feature .csv
Now open [http://localhost:3000/](http://localhost:3000/) and import the .csv file by clicking the database. After successful creation of the table, come back here. Now open [http://localhost:3000/](http://localhost:3000/) and import the .csv file by clicking the database. After successful creation of the table, come back here.
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment