Skip to content
Snippets Groups Projects
Unverified Commit 7d7b0a10 authored by Martin Weise's avatar Martin Weise
Browse files

Modified the query site

parent 3444514b
No related branches found
No related tags found
2 merge requests!81New stable release,!80Multiple features connected with user management and ownership of databases
This commit is part of merge request !81. Comments created here will be created in the context of that merge request.
Show whitespace changes
Inline Side-by-side
.invenio/.gitignore
+ 3
− 1
Edit View file @ 7d7b0a10
__pycache__
features.csv
venv/
\ No newline at end of file
.invenio/deposit.ipynb
+ 13
− 13
Edit
View file @ 7d7b0a10
%% Cell type:markdown id: tags:
# Deposit
Please create an account at [http://localhost:3000/register](http://localhost:3000/register) with `user:user` before executing and deposit the Invenio API key at [http://localhost:3000/user](http://localhost:3000/user).
%% Cell type:code id: tags:
``` python
import os.path
from os import listdir
from os.path import isfile, join
from api_document.api.document_endpoint_api import DocumentEndpointApi
from api_document.api.file_endpoint_api import FileEndpointApi
from api_authentication.api.authentication_endpoint_api import AuthenticationEndpointApi
authentication = AuthenticationEndpointApi()
document = DocumentEndpointApi()
file = FileEndpointApi()
```
%% Cell type:markdown id: tags:
1. Obtain an authentication token
%% Cell type:code id: tags:
``` python
response = authentication.authenticate_user1({
"username": "user",
"password": "user"
})
document.api_client.default_headers = {"Authorization": "Bearer " + response.token}
file.api_client.default_headers = {"Authorization": "Bearer " + response.token}
```
%% Cell type:markdown id: tags:
2. Create a metadata draft
%% Cell type:code id: tags:
``` python
response = document.create({
"access": {
"record": "public",
"files": "public"
},
"files": {
"enabled": True
},
"metadata": {
"creators": [
{
"affiliations": [
{
"name": "TU Wien"
}
],
"person_or_org": {
"type": "personal",
"name": "M., Weise",
"identifiers": [
{
"scheme": "orcid",
"identifier": "0000-0003-4216-302X"
}
],
"given_name": "Martin",
"family_name": "Weise"
}
}
],
"title": "Jupyter Notebook Test",
"resource_type": {
"id": "other"
},
"publication_date": "2022-06-28"
}
})
document_id = response.id
print(response)
```
%% Output
{'access': {'embargo': {'active': False, 'reason': None, 'until': None},
'files': 'public',
'record': 'public'},
'created': datetime.datetime(2022, 7, 6, 14, 20, 22, 248007, tzinfo=tzutc()),
'expires_at': datetime.datetime(2022, 7, 6, 14, 20, 22, 248035),
'created': datetime.datetime(2022, 7, 7, 17, 35, 22, 843601, tzinfo=tzutc()),
'expires_at': datetime.datetime(2022, 7, 7, 17, 35, 22, 843638),
'files': {'default_preview': None, 'enabled': True},
'id': 'dc4zh-9ce78',
'id': 'rn3hz-khe04',
'is_published': False,
'metadata': {'creators': [{'affiliations': [{'id': None, 'name': 'TU Wien'}],
'person_or_org': {'family_name': 'Weise',
'given_name': 'Martin',
'identifiers': [{'identifier': '0000-0003-4216-302X',
'scheme': 'orcid'}],
'name': 'Weise, Martin',
'type': 'personal'},
'role': None}],
'publication_date': datetime.datetime(2022, 6, 28, 0, 0),
'resource_type': {'id': 'other'},
'title': 'Jupyter Notebook Test'},
'pids': {'doi': None},
'revision_id': 4,
'updated': datetime.datetime(2022, 7, 6, 14, 20, 22, 266432, tzinfo=tzutc()),
'updated': datetime.datetime(2022, 7, 7, 17, 35, 22, 867664, tzinfo=tzutc()),
'versions': {'index': 1, 'is_latest': False, 'is_latest_draft': True}}
%% Cell type:markdown id: tags:
3. Upload the files
%% Cell type:code id: tags:
``` python
files = [f for f in listdir("./audio") if isfile(join("./audio", f))]
for f in files:
print("... upload file", "/tmp/" + f)
response = file.upload_file({
"location": os.path.curdir + "/tmp/" + f
}, document_id)
```
%% Output
... upload file /tmp/colive.0044_20200518133554_1_m4a_1.wav
... upload file /tmp/colive.0044_20200518133554_2_m4a_1.wav
... upload file /tmp/colive.0066_20200611134530_1_m4a_0.wav
... upload file /tmp/colive.0066_20200611134530_2_m4a_0.wav
... upload file /tmp/colive.0066_20200612072315_1_m4a_0.wav
%% Cell type:markdown id: tags:
4. Publish the record
%% Cell type:code id: tags:
``` python
response = document.publish(document_id)
print(response)
```
%% Output
{'access': {'embargo': {'active': False, 'reason': None, 'until': None},
'files': 'public',
'record': 'public'},
'created': datetime.datetime(2022, 7, 6, 14, 20, 28, 574632, tzinfo=tzutc()),
'created': datetime.datetime(2022, 7, 7, 17, 35, 24, 218701, tzinfo=tzutc()),
'expires_at': None,
'files': {'default_preview': None, 'enabled': True},
'id': 'dc4zh-9ce78',
'id': 'rn3hz-khe04',
'is_published': True,
'metadata': {'creators': [{'affiliations': [{'id': None, 'name': 'TU Wien'}],
'person_or_org': {'family_name': 'Weise',
'given_name': 'Martin',
'identifiers': [{'identifier': '0000-0003-4216-302X',
'scheme': 'orcid'}],
'name': 'Weise, Martin',
'type': 'personal'},
'role': None}],
'publication_date': datetime.datetime(2022, 6, 28, 0, 0),
'resource_type': {'id': 'other'},
'title': 'Jupyter Notebook Test'},
'pids': {'doi': {'client': 'datacite',
'identifier': '10.70124/dc4zh-9ce78',
'identifier': '10.70124/rn3hz-khe04',
'provider': 'datacite'}},
'revision_id': 3,
'updated': datetime.datetime(2022, 7, 6, 14, 20, 28, 675164, tzinfo=tzutc()),
'updated': datetime.datetime(2022, 7, 7, 17, 35, 24, 298101, tzinfo=tzutc()),
'versions': {'index': 1, 'is_latest': True, 'is_latest_draft': True}}
......
.invenio/feature_extract.ipynb
+ 15
− 15
Edit
View file @ 7d7b0a10
%% Cell type:markdown id: tags:
# Feature Extraction & Deposit
In this notebook we define an example of creating a database from a .csv and perform feature extraction of audio files. The APIs are auto-generated from the Swagger Endpoint documentations using [`generate.sh`](https://gitlab.phaidra.org/fair-data-austria-db-repository/fda-docs/-/blob/master/swagger/generate.sh). Steps we perform:
1. Download a music file from a public repository
2. Perform feature extraction
3. Obtain an authentication token
4. Create a mariadb container
5. Start the mariadb container
6. Create a database within the mariadb container
7. Import the feature .csv (manually)
Please create an account at [http://localhost:3000/register](http://localhost:3000/register) with `user:user` before executing.
%% Cell type:code id: tags:
``` python
import os.path
import uuid
import time
import re
import csv
import requests as rq
from api_authentication.api.authentication_endpoint_api import AuthenticationEndpointApi
from api_authentication.api.user_endpoint_api import UserEndpointApi
from api_container.api.container_endpoint_api import ContainerEndpointApi
from api_database.api.container_database_endpoint_api import ContainerDatabaseEndpointApi
from api_table.api.table_endpoint_api import TableEndpointApi
authentication = AuthenticationEndpointApi()
user = UserEndpointApi()
container = ContainerEndpointApi()
database = ContainerDatabaseEndpointApi()
table = TableEndpointApi()
doi = "10.5281/zenodo.5649276"
email = "some@example.com"
```
%% Cell type:markdown id: tags:
9. Download wav
Resolve the DOI to URI
%% Cell type:code id: tags:
``` python
response = rq.get("https://doi.org/" + doi)
id = re.findall("/([a-z0-9-]+)$", response.url)[0]
host = re.findall("^https?:\/\/([a-z0-9]+\.[a-z]+)", response.url)[0]
print("Resolved DOI to", host, "and record id", id)
```
%% Output
Resolved DOI to zenodo.org and record id 5649276
%% Cell type:markdown id: tags:
2. Perform feature extraction
%% Cell type:code id: tags:
``` python
response = rq.get("https://" + host + "/api/records/" + id)
record = response.json()
i = 0
with open(os.path.expanduser("~/features.csv"), "w") as f:
writer = csv.writer(f)
writer.writerow(["key", "size", "link"])
for file in record["files"]:
rq.get(file["links"]["self"])
print("... feature extract from", file["links"]["self"])
writer.writerow([file["key"], file["size"], file["links"]["self"]])
i += 1
if i > 5:
break
print("Generated a feature .csv in your home directory")
```
%% Output
... feature extract from https://zenodo.org/api/files/22d69a63-2aff-47ae-b818-be78a23e9889/colive.0044_20200518133554_1_m4a_1.wav
... feature extract from https://zenodo.org/api/files/22d69a63-2aff-47ae-b818-be78a23e9889/colive.0044_20200518133554_2_m4a_1.wav
... feature extract from https://zenodo.org/api/files/22d69a63-2aff-47ae-b818-be78a23e9889/colive.0066_20200611134530_1_m4a_0.wav
... feature extract from https://zenodo.org/api/files/22d69a63-2aff-47ae-b818-be78a23e9889/colive.0066_20200611134530_2_m4a_0.wav
... feature extract from https://zenodo.org/api/files/22d69a63-2aff-47ae-b818-be78a23e9889/colive.0066_20200612072315_1_m4a_0.wav
... feature extract from https://zenodo.org/api/files/22d69a63-2aff-47ae-b818-be78a23e9889/colive.0066_20200612072315_2_m4a_0.wav
Generated a feature .csv in your home directory
%% Cell type:markdown id: tags:
3. Obtain an authentication token
%% Cell type:code id: tags:
``` python
response = authentication.authenticate_user1({
"username": "user",
"password": "user"
})
container.api_client.default_headers = {"Authorization": "Bearer " + response.token}
database.api_client.default_headers = {"Authorization": "Bearer " + response.token}
table.api_client.default_headers = {"Authorization": "Bearer " + response.token}
```
%% Cell type:markdown id: tags:
4. Create a mariadb container
%% Cell type:code id: tags:
``` python
response = container.create1({
"name": "MIR " + str(uuid.uuid1()),
"repository": "mariadb",
"tag": "10.5"
})
container_id = response.id
print(response)
```
%% Output
{'hash': 'a431232a07efc4027cf71beaf38e7465984985ded1d4a67a94d092b49bd4d65e',
'id': 1,
'internal_name': 'fda-userdb-mir-529d42b0-f796-11ec-ad95-64bc58900b78',
{'hash': 'aed197f5575f9bff89501d4fa7f58b0fc19b018848cc2e1c1c15388106a78ad0',
'id': 2,
'internal_name': 'fda-userdb-mir-783dbf3c-fe1b-11ec-aa77-8c8caada74c3',
'is_public': None,
'name': 'MIR 529d42b0-f796-11ec-ad95-64bc58900b78'}
'name': 'MIR 783dbf3c-fe1b-11ec-aa77-8c8caada74c3'}
%% Cell type:markdown id: tags:
5. Start the mariadb container
%% Cell type:code id: tags:
``` python
response = container.modify({
"action": "START"
}, container_id)
time.sleep(5)
print(response)
```
%% Output
{'hash': 'a431232a07efc4027cf71beaf38e7465984985ded1d4a67a94d092b49bd4d65e',
'id': 1,
'internal_name': 'fda-userdb-mir-529d42b0-f796-11ec-ad95-64bc58900b78',
{'hash': 'aed197f5575f9bff89501d4fa7f58b0fc19b018848cc2e1c1c15388106a78ad0',
'id': 2,
'internal_name': 'fda-userdb-mir-783dbf3c-fe1b-11ec-aa77-8c8caada74c3',
'is_public': None,
'name': 'MIR 529d42b0-f796-11ec-ad95-64bc58900b78'}
'name': 'MIR 783dbf3c-fe1b-11ec-aa77-8c8caada74c3'}
%% Cell type:markdown id: tags:
6. Create a database within the mariadb container
%% Cell type:code id: tags:
``` python
response = database.create({
"name": "MIR " + str(uuid.uuid1()),
"description": "Music Information Retrieval",
"is_public": True
}, container_id)
database_id = response.id
```
%% Cell type:markdown id: tags:
7. Import the feature .csv
Now open [http://localhost:3000/](http://localhost:3000/) and import the .csv file by clicking the database. After successful creation of the table, come back here.
......
fda-ui/pages/container/_container_id/database/_database_id/query/_query_id/index.vue
+ 8
− 2
Edit View file @ 7d7b0a10
......@@ -152,7 +152,12 @@
</v-list-item-title>
<v-list-item-content>
<v-skeleton-loader v-if="loadingQuery" type="text" class="skeleton-small" />
<span v-if="!loadingQuery">{{ creator }}</span>
<span v-if="!loadingQuery">
{{ creator }}
<sup>
<v-icon v-if="database.creator.email_verified" small color="primary">mdi-check-decagram</v-icon>
</sup>
</span>
</v-list-item-content>
<v-list-item-title class="mt-2">
Query Execution
......@@ -270,7 +275,8 @@ export default {
is_public: null,
publisher: null,
creator: {
username: null
username: null,
email_verified: false
},
license: {
identifier: null,
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment