Skip to content
Snippets Groups Projects
Commit 90049cef authored by Martin Weise's avatar Martin Weise
Browse files

Merge branch '396-fix-the-unit-independent-search' into 'dev' 

Resolve "Fix the unit independent search"

See merge request !216
parents 3af625c4 f3cc916b
No related branches found
No related tags found
4 merge requests!231CI: Remove build for log-service,!228Better error message handling in the frontend,!223Release of version 1.4.0,!216Resolve "Fix the unit independent search"
Expand all Show whitespace changes
Inline Side-by-side
Showing
with 45181 additions and 78 deletions
dbrepo-search-service/Dockerfile
+ 1
− 0
View file @ 90049cef
...@@ -11,6 +11,7 @@ RUN pip install pipenv && \ ...@@ -11,6 +11,7 @@ RUN pip install pipenv && \
pipenv install --system --deploy pipenv install --system --deploy
COPY ./app ./app COPY ./app ./app
COPY ./omlib ./omlib
COPY ./scripts ./scripts COPY ./scripts ./scripts
COPY ./us-yml ./us-yml COPY ./us-yml ./us-yml
COPY config.py wsgi.py ./ COPY config.py wsgi.py ./
......
dbrepo-search-service/Pipfile
+ 1
− 0
View file @ 90049cef
...@@ -16,6 +16,7 @@ python-dotenv = "~=1.0" ...@@ -16,6 +16,7 @@ python-dotenv = "~=1.0"
sqlalchemy-utils = "*" sqlalchemy-utils = "*"
testcontainers-opensearch = "*" testcontainers-opensearch = "*"
pytest = "*" pytest = "*"
rdflib = "*"
[dev-packages] [dev-packages]
......
dbrepo-search-service/Pipfile.lock
+ 28
− 4
View file @ 90049cef
{ {
"_meta": { "_meta": {
"hash": { "hash": {
"sha256": "d126ace88662c624b1574018644daa91049070c38a8aabb0c48aed5257f0a973" "sha256": "c99a5f14ded92b79ae4f023dc42daf7fc9d6a89381b43dfc91dfa5c053b253ac"
}, },
"pipfile-spec": 6, "pipfile-spec": 6,
"requires": { "requires": {
...@@ -284,11 +284,11 @@ ...@@ -284,11 +284,11 @@
}, },
"idna": { "idna": {
"hashes": [ "hashes": [
"sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4", "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca",
"sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2" "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"
], ],
"markers": "python_version >= '3.5'", "markers": "python_version >= '3.5'",
"version": "==3.4" "version": "==3.6"
}, },
"iniconfig": { "iniconfig": {
"hashes": [ "hashes": [
...@@ -298,6 +298,13 @@ ...@@ -298,6 +298,13 @@
"markers": "python_version >= '3.7'", "markers": "python_version >= '3.7'",
"version": "==2.0.0" "version": "==2.0.0"
}, },
"isodate": {
"hashes": [
"sha256:0751eece944162659049d35f4f549ed815792b38793f07cf73381c1c87cbed96",
"sha256:48c5881de7e8b0a0d648cb024c8062dc84e7b840ed81e864c7614fd3c127bde9"
],
"version": "==0.6.1"
},
"itsdangerous": { "itsdangerous": {
"hashes": [ "hashes": [
"sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44", "sha256:2c2349112351b88699d8d4b6b075022c0808887cb7ad10069318a8b0bc88db44",
...@@ -453,6 +460,14 @@ ...@@ -453,6 +460,14 @@
"markers": "python_version >= '3.7'", "markers": "python_version >= '3.7'",
"version": "==2.8.0" "version": "==2.8.0"
}, },
"pyparsing": {
"hashes": [
"sha256:32c7c0b711493c72ff18a981d24f28aaf9c1fb7ed5e9667c9e84e3db623bdbfb",
"sha256:ede28a1a32462f5a9705e07aea48001a08f7cf81a021585011deba701581a0db"
],
"markers": "python_full_version >= '3.6.8'",
"version": "==3.1.1"
},
"pytest": { "pytest": {
"hashes": [ "hashes": [
"sha256:0d009c083ea859a71b76adf7c1d502e4bc170b80a8ef002da5806527b9591fac", "sha256:0d009c083ea859a71b76adf7c1d502e4bc170b80a8ef002da5806527b9591fac",
...@@ -535,6 +550,15 @@ ...@@ -535,6 +550,15 @@
"markers": "python_version >= '3.6'", "markers": "python_version >= '3.6'",
"version": "==6.0.1" "version": "==6.0.1"
}, },
"rdflib": {
"hashes": [
"sha256:0438920912a642c866a513de6fe8a0001bd86ef975057d6962c79ce4771687cd",
"sha256:9995eb8569428059b8c1affd26b25eac510d64f5043d9ce8c84e0d0036e995ae"
],
"index": "pypi",
"markers": "python_full_version >= '3.8.1' and python_full_version < '4.0.0'",
"version": "==7.0.0"
},
"referencing": { "referencing": {
"hashes": [ "hashes": [
"sha256:381b11e53dd93babb55696c71cf42aef2d36b8a150c49bf0bc301e36d536c882", "sha256:381b11e53dd93babb55696c71cf42aef2d36b8a150c49bf0bc301e36d536c882",
......
dbrepo-search-service/app/api/routes.py
+ 5
− 2
View file @ 90049cef
...@@ -158,6 +158,9 @@ def search(): ...@@ -158,6 +158,9 @@ def search():
t2 = req_body.get("t2") t2 = req_body.get("t2")
if not str(t2).isdigit(): if not str(t2).isdigit():
t2 = None t2 = None
fieldValuePairs = req_body.get("field_value_pairs") field_value_pairs = req_body.get("field_value_pairs")
response = general_search(search_term, t1, t2, fieldValuePairs) if t1 is not None and t2 is not None and "unit.uri" in field_value_pairs and "concept.uri" in field_value_pairs:
response = unit_independent_search(t1, t2, field_value_pairs)
else:
response = general_search(search_term, t1, t2, field_value_pairs)
return response, 200 return response, 200
dbrepo-search-service/app/opensearch_client.py
+ 120
− 66
View file @ 90049cef
""" """
The opensearch_client.py is used by the different API endpoints in routes.py to handle requests to the opensearch db The opensearch_client.py is used by the different API endpoints in routes.py to handle requests to the opensearch db
""" """
import json
import logging import logging
import re import re
from flask import current_app from flask import current_app
from collections.abc import MutableMapping from collections.abc import MutableMapping
from omlib.dimension import Dimension
from omlib.measure import om
from omlib.constants import SI, OM_IDS
from omlib.omconstants import OM
from omlib.unit import Unit
def flatten_dict( def flatten_dict(
d: MutableMapping, parent_key: str = "", sep: str = "." d: MutableMapping, parent_key: str = "", sep: str = "."
...@@ -123,42 +130,9 @@ def general_search(search_term=None, t1=None, t2=None, fieldValuePairs=None): ...@@ -123,42 +130,9 @@ def general_search(search_term=None, t1=None, t2=None, fieldValuePairs=None):
:param value: the value the specified field should match :param value: the value the specified field should match
:return: :return:
""" """
logging.info(f"Performing general search")
searchable_indices = ["database", "user", "table", "column", "identifier", "view", "concept", "unit"] searchable_indices = ["database", "user", "table", "column", "identifier", "view", "concept", "unit"]
index = searchable_indices index = searchable_indices
# field_list = [
# "id",
# "internal_name",
# "table.name",
# "database.is_public",
# "database.container.image.name",
# "database.container.image.version",
# "table.description",
# "identifier.titles.title",
# "identifier.descriptions.description",
# "identifier.publisher",
# "identifier.creators.*.firstname",
# "identifier.creators.*.lastname",
# "identifier.creators.*.creator_name",
# "column.column_type",
# "column.is_null_allowed",
# "column.is_primary_key",
# "unit.uri",
# "unit.name",
# "unit.description",
# "concept.uri",
# "concept.name",
# "concept.description",
# "funders",
# "title",
# "description",
# "creator.username",
# "author",
# "name",
# "uri",
# "database.*",
# "internal_name",
# "is_public",
# ]
queries = [] queries = []
if search_term is not None: if search_term is not None:
logging.debug('query has search_term present') logging.debug('query has search_term present')
...@@ -195,6 +169,7 @@ def general_search(search_term=None, t1=None, t2=None, fieldValuePairs=None): ...@@ -195,6 +169,7 @@ def general_search(search_term=None, t1=None, t2=None, fieldValuePairs=None):
is_range_query = True is_range_query = True
logging.debug(f"query has start value {t1} and end value {t2} present") logging.debug(f"query has start value {t1} and end value {t2} present")
for key, value in fieldValuePairs.items(): for key, value in fieldValuePairs.items():
logging.debug(f"current key={key}, value={value}")
if key == "type" and value in searchable_indices: if key == "type" and value in searchable_indices:
logging.debug("search for specific index: %s", value) logging.debug("search for specific index: %s", value)
index = value index = value
...@@ -231,7 +206,8 @@ def general_search(search_term=None, t1=None, t2=None, fieldValuePairs=None): ...@@ -231,7 +206,8 @@ def general_search(search_term=None, t1=None, t2=None, fieldValuePairs=None):
} }
}) })
elif is_range_query and re.match(f"unit\.", key): elif is_range_query and re.match(f"unit\.", key):
logging.debug(f"omit key={key} because query type=full range and key is somewhat unit") logging.debug(
f"omit key={key} because query type=full range and key is somewhat unit")
logging.info(f"add match-query for range [{t1},{t2}]") logging.info(f"add match-query for range [{t1},{t2}]")
musts.append({ musts.append({
"range": { "range": {
...@@ -249,7 +225,7 @@ def general_search(search_term=None, t1=None, t2=None, fieldValuePairs=None): ...@@ -249,7 +225,7 @@ def general_search(search_term=None, t1=None, t2=None, fieldValuePairs=None):
}) })
else: else:
precision = "90%" precision = "90%"
if key in ["attributes.orcid", "creators.name_identifier"]: if key in ["attributes.orcid", "creators.name_identifier", "concept.uri"]:
precision = "100%" precision = "100%"
logging.debug(f"key {key} needs precision of 100%") logging.debug(f"key {key} needs precision of 100%")
musts.append({ musts.append({
...@@ -261,36 +237,6 @@ def general_search(search_term=None, t1=None, t2=None, fieldValuePairs=None): ...@@ -261,36 +237,6 @@ def general_search(search_term=None, t1=None, t2=None, fieldValuePairs=None):
queries.append(specific_query) queries.append(specific_query)
body = { body = {
"query": {"bool": {"must": queries}} "query": {"bool": {"must": queries}}
# "_source": [
# "_class",
# "id",
# "table_id",
# "database_id",
# "name",
# "identifier.*",
# "column_type",
# "description",
# "titles",
# "descriptions",
# "funders",
# "licenses",
# "creators",
# "visibility",
# "title",
# "type",
# "uri",
# "username",
# "is_public",
# "created",
# "_score",
# "concept",
# "unit",
# "author",
# "docID",
# "creator.*",
# "owner.*",
# "details.*",
# ],
} }
logging.debug('search index: %s', index) logging.debug('search index: %s', index)
logging.debug('search body: %s', body) logging.debug('search body: %s', body)
...@@ -301,3 +247,111 @@ def general_search(search_term=None, t1=None, t2=None, fieldValuePairs=None): ...@@ -301,3 +247,111 @@ def general_search(search_term=None, t1=None, t2=None, fieldValuePairs=None):
response["status"] = 200 response["status"] = 200
# response = [hit["_source"] for hit in response["hits"]["hits"]] # response = [hit["_source"] for hit in response["hits"]["hits"]]
return response return response
def flatten(mylist):
return [item for sublist in mylist for item in sublist]
def unit_uri_to_unit(uri):
base_identifier = uri[len(OM_IDS.NAMESPACE):].replace("-", "")
return getattr(OM, base_identifier)
def unit_independent_search(t1=None, t2=None, field_value_pairs=None):
"""
Main method for seaching stuff in the opensearch db
all parameters are optional
:param t1: start value
:param t2: end value
:param field_value_pairs: the key-value pairs
:return:
"""
logging.info(f"Performing unit-independent search")
searches = []
response = current_app.opensearch_client.search(
index="column",
body={
"size": 0,
"aggs": {
"units": {
"terms": {"field": "unit.uri", "size": 500}
}
}
}
)
unit_uris = [hit["key"] for hit in response["aggregations"]["units"]["buckets"]]
logging.debug(f"found {len(unit_uris)} unit(s) in column index")
base_unit = unit_uri_to_unit(field_value_pairs["unit.uri"])
for unit_uri in unit_uris:
gte = t1
lte = t2
if unit_uri != field_value_pairs["unit.uri"]:
target_unit = unit_uri_to_unit(unit_uri)
if not Unit.can_convert(base_unit, target_unit):
logging.error(f"Cannot convert unit {field_value_pairs['unit.uri']} to target unit {unit_uri}")
continue
gte = om(t1, base_unit).convert(target_unit)
lte = om(t2, base_unit).convert(target_unit)
logging.debug(
f"converted original range [{t1},{t2}] for base unit {base_unit} to mapped range [{gte},{lte}] for target unit={target_unit}")
searches.append({'index': 'column'})
searches.append({
"query": {
"bool": {
"must": [
{
"match": {
"concept.uri": {
"query": field_value_pairs["concept.uri"]
}
}
},
{
"range": {
"val_min": {
"gte": gte
}
}
},
{
"range": {
"val_max": {
"lte": lte
}
}
},
{
"match": {
"unit.uri": {
"query": unit_uri
}
}
}
]
}
}
})
# searches.append({'index': 'column'})
# searches.append({
# "query": {
# "match_all": {}
# }
# })
logging.debug('searches: %s', searches)
body = ''
for search in searches:
body += '%s \n' % json.dumps(search)
responses = current_app.opensearch_client.msearch(
body=body
)
response = {
"hits": {
"hits": flatten([hits["hits"]["hits"] for hits in responses["responses"]])
},
"took": responses["took"],
"status": 200
}
return response
dbrepo-search-service/omlib/__init__.py 0 → 100644
+ 0
− 0
View file @ 90049cef
dbrepo-search-service/omlib/constants.py 0 → 100644
+ 80
− 0
View file @ 90049cef
import rdflib
from rdflib import URIRef
from omlib.dimension import Dimension
from omlib.scale import Scale
from omlib.unit import Prefix, Unit
class OM_IDS:
NAMESPACE = 'http://www.ontology-of-units-of-measure.org/resource/om-2/'
class SI:
SYSTEM_OF_UNITS = str(OM_IDS.NAMESPACE + 'InternationalSystemOfUnits') # WARNING If you change "SI" also change in constants.py
# SI Prefixes
YOTTA = Prefix('yotta', 'Y', 1e24, OM_IDS.NAMESPACE + 'yotta')
ZETTA = Prefix('zetta', 'Z', 1e21, OM_IDS.NAMESPACE + 'zetta')
EXA = Prefix('exa', 'E', 1e18, OM_IDS.NAMESPACE + 'exa')
PETA = Prefix('peta', 'P', 1e15, OM_IDS.NAMESPACE + 'peta')
TERA = Prefix('tera', 'T', 1e12, OM_IDS.NAMESPACE + 'tera')
GIGA = Prefix('giga', 'G', 1e9, OM_IDS.NAMESPACE + 'giga')
MEGA = Prefix('mega', 'M', 1e6, OM_IDS.NAMESPACE + 'mega')
KILO = Prefix('kilo', 'k', 1e3, OM_IDS.NAMESPACE + 'kilo')
HECTO = Prefix('hecto', 'h', 1e2, OM_IDS.NAMESPACE + 'hecto')
DECA = Prefix('deca', 'da', 1e1, OM_IDS.NAMESPACE + 'deca')
DECI = Prefix('deci', 'd', 1e-1, OM_IDS.NAMESPACE + 'deci')
CENTI = Prefix('centi', 'c', 1e-2, OM_IDS.NAMESPACE + 'centi')
MILLI = Prefix('milli', 'm', 1e-3, OM_IDS.NAMESPACE + 'milli')
MICRO = Prefix('micro', 'μ', 1e-6, OM_IDS.NAMESPACE + 'micro')
NANO = Prefix('nano', 'n', 1e-9, OM_IDS.NAMESPACE + 'nano')
PICO = Prefix('pico', 'p', 1e-12, OM_IDS.NAMESPACE + 'pico')
FEMTO = Prefix('femto', 'f', 1e-15, OM_IDS.NAMESPACE + 'femto')
ATTO = Prefix('atto', 'a', 1e-18, OM_IDS.NAMESPACE + 'atto')
ZEPTO = Prefix('zepto', 'z', 1e-21, OM_IDS.NAMESPACE + 'zepto')
YOCTO = Prefix('yocto', 'y', 1e-24, OM_IDS.NAMESPACE + 'yocto')
# SI Base Units
SECOND = Unit.get_singular_unit('second', 's', Dimension(1, 0, 0, 0, 0, 0, 0),
identifier=OM_IDS.NAMESPACE + 'second-Time',
system_of_units=SYSTEM_OF_UNITS, is_base_unit=True)
METRE = Unit.get_singular_unit('metre', 'm', Dimension(0, 1, 0, 0, 0, 0, 0), identifier=OM_IDS.NAMESPACE + 'metre',
system_of_units=SYSTEM_OF_UNITS, is_base_unit=True)
GRAM = Unit.get_singular_unit('gram', 'g', Dimension(0, 0, 1, 0, 0, 0, 0), identifier=OM_IDS.NAMESPACE + 'gram')
KILOGRAM = Unit.get_prefixed_unit(KILO, GRAM, identifier=OM_IDS.NAMESPACE + 'kilogram',
system_of_units=SYSTEM_OF_UNITS,
is_base_unit=True)
AMPERE = Unit.get_singular_unit('ampere', 'A', Dimension(0, 0, 0, 1, 0, 0, 0),
identifier=OM_IDS.NAMESPACE + 'ampere',
system_of_units=SYSTEM_OF_UNITS, is_base_unit=True)
KELVIN = Unit.get_singular_unit('kelvin', 'K', Dimension(0, 0, 0, 0, 1, 0, 0),
identifier=OM_IDS.NAMESPACE + 'kelvin',
system_of_units=SYSTEM_OF_UNITS, is_base_unit=True)
MOLE = Unit.get_singular_unit('mole', 'mol', Dimension(0, 0, 0, 0, 0, 1, 0),
identifier=OM_IDS.NAMESPACE + 'mole',
system_of_units=SYSTEM_OF_UNITS, is_base_unit=True)
CANDELA = Unit.get_singular_unit('candela', 'cd', Dimension(0, 0, 0, 0, 0, 0, 1),
identifier=OM_IDS.NAMESPACE + 'candela', system_of_units=SYSTEM_OF_UNITS,
is_base_unit=True)
class IEC:
KIBI = Prefix('kibi', 'Ki', pow(2, 10), OM_IDS.NAMESPACE + 'kibi')
MEBI = Prefix('mebi', 'Mi', pow(2, 20), OM_IDS.NAMESPACE + 'mebi')
GIBI = Prefix('gibi', 'Gi', pow(2, 30), OM_IDS.NAMESPACE + 'gibi')
TEBI = Prefix('tebi', 'Ti', pow(2, 40), OM_IDS.NAMESPACE + 'tebi')
PEBI = Prefix('pebi', 'Pi', pow(2, 50), OM_IDS.NAMESPACE + 'pebi')
EXBI = Prefix('exbi', 'Ei', pow(2, 60), OM_IDS.NAMESPACE + 'exbi')
ZEBI = Prefix('zebi', 'Zi', pow(2, 70), OM_IDS.NAMESPACE + 'zebi')
YOBI = Prefix('yobi', 'Yi', pow(2, 80), OM_IDS.NAMESPACE + 'yobi')
class JEDEC:
KILO = Prefix('kilo', 'k', pow(2, 10), OM_IDS.NAMESPACE + 'jedec-kilo')
MEGA = Prefix('mega', 'M', pow(2, 20), OM_IDS.NAMESPACE + 'jedec-mega')
GIGA = Prefix('giga', 'G', pow(2, 30), OM_IDS.NAMESPACE + 'jedec-giga')
dbrepo-search-service/omlib/dimension.py 0 → 100644
+ 69
− 0
View file @ 90049cef
from omlib.exceptions.dimensionexception import DimensionalException
class Dimension:
def __init__(self, T=0, L=0, M=0, I=0, Theta=0, N=0, J=0):
self.T = T
self.L = L
self.M = M
self.I = I
self.Theta = Theta
self.N = N
self.J = J
def time_dimension_exponent(self):
return self.T
def length_dimension_exponent(self):
return self.L
def mass_dimension_exponent(self):
return self.M
def electric_current_dimension_exponent(self):
return self.I
def thermodynamic_temperature_dimension_exponent(self):
return self.Theta
def amount_of_substance_dimension_exponent(self):
return self.N
def luminous_intensity_dimension_exponent(self):
return self.J
def __eq__(self, other):
return (self.T == other.T and self.L == other.L and self.M == other.M and self.I == other.I and
self.Theta == other.Theta and self.N == other.N and self.J == other.J)
def __str__(self):
return f'(T={self.T}, L={self.L}, M={self.M}, I={self.I}, θ={self.Theta}, N={self.N}, J={self.J})'
def __add__(self, other):
if not self == other:
raise DimensionalException("Entities of different dimensions cannot be added together. {} != {}"
.format(self, other))
return self
def __sub__(self, other):
if not self == other:
raise DimensionalException("Entities of different dimensions cannot be subtracted from each other. {} != {}"
.format(self, other))
return self
def __mul__(self, other):
return Dimension(self.T + other.T, self.L + other.L, self.M + other.M, self.I + other.I,
self.Theta + other.Theta, self.N + other.N, self.J + other.J)
def __truediv__(self, other):
return Dimension(self.T - other.T, self.L - other.L, self.M - other.M, self.I - other.I,
self.Theta - other.Theta, self.N - other.N, self.J - other.J)
@staticmethod
def pow(base, exponent):
if isinstance(base, Dimension):
return Dimension(base.T * exponent, base.L * exponent, base.M * exponent, base.I * exponent,
base.Theta * exponent, base.N * exponent, base.J * exponent)
else:
return Dimension()
dbrepo-search-service/omlib/exceptions/__init__.py 0 → 100644
+ 0
− 0
View file @ 90049cef
dbrepo-search-service/omlib/exceptions/dimensionexception.py 0 → 100644
+ 4
− 0
View file @ 90049cef
class DimensionalException(Exception):
def __init__(self, message):
self.message = message
dbrepo-search-service/omlib/exceptions/unitconversionexception.py 0 → 100644
+ 10
− 0
View file @ 90049cef
class UnitConversionException(Exception):
def __init__(self, message):
self.message = message
class ScaleConversionException(Exception):
def __init__(self, message):
self.message = message
dbrepo-search-service/omlib/exceptions/unitidentityexception.py 0 → 100644
+ 10
− 0
View file @ 90049cef
class UnitIdentityException(Exception):
def __init__(self, message):
self.message = message
class ScaleIdentityException(Exception):
def __init__(self, message):
self.message = message
dbrepo-search-service/omlib/measure.py 0 → 100644
+ 326
− 0
View file @ 90049cef
import math
from omlib.constants import SI
from omlib.exceptions.dimensionexception import DimensionalException
from omlib.scale import Scale
from omlib.thing import Thing
from omlib.unit import Unit, PrefixedUnit, SingularUnit
def om(numerical_value, unit_or_scale, identifier=None):
if isinstance(unit_or_scale, Unit):
return Measure(numerical_value, unit_or_scale, identifier)
if isinstance(unit_or_scale, Scale):
return Point(numerical_value, unit_or_scale, identifier)
return None
class Point(Thing):
@staticmethod
def create_by_converting(point, to_scale):
if not isinstance(point, Point):
raise ValueError("The parameter to the convert method is not of the correct type (Point).")
if not isinstance(to_scale, Scale):
raise ValueError("The parameter to the convert method is not of the correct type (Scale).")
new_point = Point(point.numericalValue, point.scale)
new_point.convert(to_scale)
return new_point
@staticmethod
def create_by_converting_to_ratio_scale(point):
if not isinstance(point, Point):
raise ValueError("The parameter to the convert method is not of the correct type (Point).")
new_point = Point(point.numericalValue, point.scale)
new_point.convert_to_ratio_scale()
return new_point
def __init__(self, numerical_value, scale, identifier=None):
super().__init__(identifier=identifier)
self.numericalValue = numerical_value
self.scale = scale
def convert(self, to_scale):
if not isinstance(to_scale, Scale):
raise ValueError("The parameter to the convert method is not of the correct type (Scale).")
factor = Scale.conversion_factor(self.scale, to_scale)
off_set = Scale.conversion_off_set(self.scale, to_scale)
self.numericalValue = self.numericalValue * factor + off_set
self.scale = to_scale
def convert_to_ratio_scale(self):
base = self.scale.base_ratio_scale()
factor = Scale.conversion_factor(self.scale, base[0])
off_set = Scale.conversion_off_set(self.scale, base[0])
self.numericalValue = self.numericalValue * factor + off_set
self.scale = base[0]
def __str__(self):
return f'{self.numericalValue} {self.scale.unit.symbol()}'
def __new_value_for_comparisson(self, other):
if isinstance(other, Point):
factor = Scale.conversion_factor(self.scale, other.scale)
off_set = Scale.conversion_off_set(self.scale, other.scale)
new_value = self.numericalValue * factor + off_set
return new_value
return None
def __eq__(self, other):
new_value = self.__new_value_for_comparisson(other)
if new_value is not None:
return new_value == other.numericalValue
return False
def __ne__(self, other):
new_value = self.__new_value_for_comparisson(other)
if new_value is not None:
return new_value != other.numericalValue
return False
def __lt__(self, other):
new_value = self.__new_value_for_comparisson(other)
if new_value is not None:
return new_value < other.numericalValue
return False
def __le__(self, other):
new_value = self.__new_value_for_comparisson(other)
if new_value is not None:
return new_value <= other.numericalValue
return False
def __gt__(self, other):
new_value = self.__new_value_for_comparisson(other)
if new_value is not None:
return new_value > other.numericalValue
return False
def __ge__(self, other):
new_value = self.__new_value_for_comparisson(other)
if new_value is not None:
return new_value >= other.numericalValue
return False
def __add__(self, other):
if not isinstance(other, Measure):
raise ValueError('The value to be added is not a measure and only measures can be added to a point.')
if not self.scale.dimensions == other.unit.dimensions:
raise DimensionalException("Measures and Points with units of different dimensions cannot be added "
"together. {} != {}"
.format(self.scale.unit, other.unit))
new_measure = Measure.create_by_converting(other, self.scale.unit)
return_point = Point(self.numericalValue + new_measure.numericalValue, self.scale)
return return_point
def __sub__(self, other):
if not isinstance(other, Measure) and not isinstance(other, Point):
raise ValueError('The value to be subtracted is not a point or a measure and only measures or points '
'can be subtracted from a point.')
if isinstance(other, Measure):
if not self.scale.dimensions == other.unit.dimensions:
raise DimensionalException("Measures and Points with units of different dimensions cannot be "
"subtracted from each other. {} != {}".format(self.scale.unit, other.unit))
new_measure = Measure.create_by_converting(other, self.scale.unit)
return_point = Point(self.numericalValue - new_measure.numericalValue, self.scale)
return return_point
if isinstance(other, Point):
if not self.scale.dimensions == other.scale.dimensions:
raise DimensionalException("Measures and Points with units of different dimensions cannot be "
"subtracted from each other. {} != {}".format(self.scale.unit, other.unit))
new_point = Point.create_by_converting(other, self.scale)
return_measure = Measure(self.numericalValue - new_point.numericalValue, self.scale.unit)
return return_measure
def __mul__(self, other):
as_measure = Measure(self.numericalValue, self.scale.unit)
return as_measure * other
def __truediv__(self, other):
as_measure = Measure(self.numericalValue, self.scale.unit)
return as_measure / other
class Measure(Thing):
@staticmethod
def create_by_converting(measure, to_unit):
if not isinstance(measure, Measure):
raise ValueError("The parameter to the convert method is not of the correct type (Measure).")
if not isinstance(to_unit, Unit):
raise ValueError("The parameter to the convert method is not of the correct type (Unit).")
new_measure = Measure(measure.numericalValue, measure.unit)
new_measure.convert(to_unit)
return new_measure
@staticmethod
def create_by_converting_to_base_units(measure, in_system_of_units=SI):
if not isinstance(measure, Measure):
raise ValueError("The parameter to the convert method is not of the correct type (Measure).")
new_measure = Measure(measure.numericalValue, measure.unit)
new_measure.convert_to_base_units(in_system_of_units)
return new_measure
@staticmethod
def create_by_converting_to_convenient_units(measure, in_system_of_units=None, use_prefixes=True):
if not isinstance(measure, Measure):
raise ValueError("The parameter to the convert method is not of the correct type (Measure).")
new_measure = Measure(measure.numericalValue, measure.unit)
new_measure.convert_to_convenient_units(in_system_of_units, use_prefixes=use_prefixes)
return new_measure
def __init__(self, numerical_value, unit, identifier=None):
super().__init__(identifier=identifier)
self.numericalValue = numerical_value
self.unit = unit
def convert(self, to_unit):
if not isinstance(to_unit, Unit):
raise ValueError("The parameter to the convert method is not of the correct type (Unit).")
factor = Unit.conversion_factor(self.unit, to_unit)
self.numericalValue = self.numericalValue * factor
self.unit = to_unit
def convert_to_base_units(self, in_system_of_units=None):
if in_system_of_units is None:
in_system_of_units = self.unit.systemOfUnits
if in_system_of_units is None:
in_system_of_units = SI.SYSTEM_OF_UNITS
base = Unit.get_base_units(self.unit, in_system_of_units)
self.convert(base)
def convert_to_convenient_units(self, system_of_units=None, use_prefixes=True):
if system_of_units is None:
system_of_units = self.unit.systemOfUnits
test_units = self.unit.with_dimensions(self.unit.dimensions, in_system_of_units=system_of_units)
selected_unit = self.unit
log_selected_value = Measure.__get_log_value(self.numericalValue, selected_unit)
for test_unit in test_units:
if use_prefixes or not isinstance(test_unit, PrefixedUnit):
factor = Unit.conversion_factor(self.unit, test_unit)
value = abs(self.numericalValue * factor)
log_value = Measure.__get_log_value(value, test_unit)
if log_value < log_selected_value:
log_selected_value = log_value
selected_unit = test_unit
if abs(log_value - log_selected_value) < 0.1 and isinstance(test_unit, SingularUnit):
log_selected_value = log_value
selected_unit = test_unit
if abs(log_value - log_selected_value) < 0.1 and isinstance(test_unit, PrefixedUnit)\
and not isinstance(selected_unit, SingularUnit):
log_selected_value = log_value
selected_unit = test_unit
self.convert(selected_unit)
@staticmethod
def __get_log_value(value, unit):
if value == 0.0:
return 0
log_value = abs(math.log10(value))
if value < 1:
log_value = log_value + 2
if not isinstance(unit, SingularUnit):
log_value = log_value + 1
return log_value
def __str__(self):
return f'{self.numericalValue} {self.unit.symbol()}'
def __new_value_for_comparisson(self, other):
if isinstance(other, Measure):
factor = Unit.conversion_factor(self.unit, other.unit)
new_value = self.numericalValue * factor
return new_value
return None
def __eq__(self, other):
new_value = self.__new_value_for_comparisson(other)
if new_value is not None:
return new_value == other.numericalValue
return False
def __ne__(self, other):
new_value = self.__new_value_for_comparisson(other)
if new_value is not None:
return new_value != other.numericalValue
return False
def __lt__(self, other):
new_value = self.__new_value_for_comparisson(other)
if new_value is not None:
return new_value < other.numericalValue
return False
def __le__(self, other):
new_value = self.__new_value_for_comparisson(other)
if new_value is not None:
return new_value <= other.numericalValue
return False
def __gt__(self, other):
new_value = self.__new_value_for_comparisson(other)
if new_value is not None:
return new_value > other.numericalValue
return False
def __ge__(self, other):
new_value = self.__new_value_for_comparisson(other)
if new_value is not None:
return new_value >= other.numericalValue
return False
def __add__(self, other):
if not isinstance(other, Measure):
raise ValueError('The value to be added is not a measure.')
if not self.unit.dimensions == other.unit.dimensions:
raise DimensionalException("Measures with units of different dimensions cannot be added together. {} != {}"
.format(self.unit, other.unit))
new_measure = Measure.create_by_converting(other, self.unit)
return_measure = Measure(self.numericalValue + new_measure.numericalValue, self.unit)
return return_measure
def __sub__(self, other):
if not isinstance(other, Measure):
raise ValueError('The value to be subtracted is not a measure.')
if not self.unit.dimensions == other.unit.dimensions:
raise DimensionalException("Measures with units of different dimensions cannot be subtracted from each "
"other. {} != {}".format(self.unit, other.unit))
new_measure = Measure.create_by_converting(other, self.unit)
return_measure = Measure(self.numericalValue - new_measure.numericalValue, self.unit)
return return_measure
def __mul__(self, other):
if isinstance(other, float) or isinstance(other, int):
new_measure = Measure(self.numericalValue * other, self.unit)
return new_measure
if not isinstance(other, Measure) and not isinstance(other, Point):
raise ValueError('The multiplicand is not a measure, a point, a float, or an int.')
other_unit = None
if isinstance(other, Measure):
other_unit = other.unit
if isinstance(other, Point):
other_unit = other.scale.unit
new_value = self.numericalValue * other.numericalValue
new_unit = Unit.get_unit_multiplication(self.unit, other_unit)
new_measure = Measure(new_value, new_unit)
simple_unit = Unit.simplified_compound_unit(new_unit)
new_measure.convert(simple_unit)
return new_measure
def __truediv__(self, other):
if isinstance(other, float) or isinstance(other, int):
new_measure = Measure(self.numericalValue / other, self.unit)
return new_measure
if not isinstance(other, Measure) and not isinstance(other, Point):
raise ValueError('The denominator is not a measure, a point, a float, or an int.')
other_unit = None
if isinstance(other, Measure):
other_unit = other.unit
if isinstance(other, Point):
other_unit = other.scale.unit
new_value = self.numericalValue / other.numericalValue
new_unit = Unit.get_unit_division(self.unit, other_unit)
new_measure = Measure(new_value, new_unit)
simple_unit = Unit.simplified_compound_unit(new_unit)
new_measure.convert(simple_unit)
return new_measure
dbrepo-search-service/omlib/omconstants.py 0 → 100644
+ 1399
− 0
View file @ 90049cef
Source diff could not be displayed: it is too large. Options to address this: view the blob.
dbrepo-search-service/omlib/rdf/__init__.py 0 → 100644
+ 0
− 0
View file @ 90049cef
dbrepo-search-service/omlib/rdf/om-2.0.rdf 0 → 100644
+ 41879
− 0
View file @ 90049cef
This diff is collapsed.
dbrepo-search-service/omlib/scale.py 0 → 100644
+ 195
− 0
View file @ 90049cef
from rdflib import URIRef
from omlib.dimension import Dimension
from omlib.exceptions.dimensionexception import DimensionalException
from omlib.exceptions.unitconversionexception import ScaleConversionException
from omlib.exceptions.unitidentityexception import ScaleIdentityException
from omlib.thing import Thing
from omlib.unit import Unit
class Scale(Thing):
_scales = []
@staticmethod
def clear_cache():
Scale._scales.clear()
@staticmethod
def _add_when_not_in_cache(scale):
for test_scale in Scale._scales:
if test_scale == scale:
if test_scale.identifier == scale.identifier or not isinstance(scale.identifier, URIRef) \
or not isinstance(test_scale.identifier, URIRef):
if isinstance(test_scale, RatioScale) and isinstance(scale, RatioScale):
if test_scale.unit == scale.unit:
if isinstance(scale.identifier, URIRef) and not isinstance(test_scale.identifier, URIRef):
test_scale.identifier = scale.identifier
return test_scale
if isinstance(test_scale, IntervalScale) and isinstance(scale, IntervalScale):
if test_scale.unit == scale.unit and test_scale.baseScale == scale.baseScale \
and test_scale.offSet == scale.offSet:
if isinstance(scale.identifier, URIRef) and not isinstance(test_scale.identifier, URIRef):
test_scale.identifier = scale.identifier
return test_scale
if test_scale.identifier == scale.identifier:
raise ScaleIdentityException("The scale with identifier {} already exists but with different"
"properties. \n{} \nis not the same as \n{}".format(scale.identifier,
scale, test_scale))
Scale._scales.append(scale)
return scale
@staticmethod
def with_label(label):
return_scales = []
for test_scale in Scale._scales:
for test_label in test_scale.allLabels:
if test_label == label:
return_scales.append(test_label)
return return_scales
@staticmethod
def with_identifier(identifier):
for test_scale in Scale._scales:
if test_scale.identifier == identifier:
return test_scale
return None
@staticmethod
def with_dimensions(dimensions, in_system_of_units=None):
return_scales = []
for test_scale in Scale._scales:
if test_scale.dimensions == dimensions and test_scale.systemOfUnits == in_system_of_units:
return_scales.append(test_scale)
return return_scales
@staticmethod
def get_ratio_scale(unit, label=None, identifier=None):
scale = RatioScale(unit, label, identifier)
scale = Scale._add_when_not_in_cache(scale)
return scale
@staticmethod
def get_interval_scale(base_scale, unit, off_set, label=None, identifier=None):
scale = IntervalScale(base_scale, unit, off_set, label, identifier)
scale = Scale._add_when_not_in_cache(scale)
return scale
@staticmethod
def conversion_factor(from_scale, to_scale):
if (isinstance(from_scale, RatioScale) or isinstance(from_scale, IntervalScale)) and \
(isinstance(to_scale, RatioScale) or isinstance(to_scale, IntervalScale)):
can_convert = Unit.can_convert(from_scale.unit, to_scale.unit)
if not can_convert:
raise DimensionalException("A scale with dimensions {} cannot be converted to a scale with "
"dimensions {}."
.format(from_scale.dimensions, to_scale.dimensions))
factor = Unit.conversion_factor(from_scale.unit, to_scale.unit)
return factor
else:
raise ScaleConversionException("Cannot convert from {} to {} as they are not both cardinal scales."
.format(from_scale, to_scale))
@staticmethod
def conversion_off_set(from_scale, to_scale):
if (isinstance(from_scale, RatioScale) or isinstance(from_scale, IntervalScale)) and \
(isinstance(to_scale, RatioScale) or isinstance(to_scale, IntervalScale)):
from_ratio_scale = from_scale.base_ratio_scale()
to_ratio_scale = to_scale.base_ratio_scale()
from_ratio_factor = Unit.conversion_factor(from_scale.unit,from_ratio_scale[0].unit)
to_ratio_factor = Unit.conversion_factor(to_scale.unit,to_ratio_scale[0].unit)
if from_ratio_scale[0] == to_ratio_scale[0]:
off_set = (to_ratio_scale[1] * to_ratio_factor - from_ratio_scale[1]*from_ratio_factor) \
/ to_ratio_factor
return off_set
else:
raise ScaleConversionException("Cannot convert from {} to {} as they do not use the same base ratio "
"scale, i.e. they do not have the same known zero point."
.format(from_scale, to_scale))
else:
raise ScaleConversionException("Cannot convert from {} to {} as they are not both cardinal scales."
.format(from_scale, to_scale))
def __init__(self, label=None, identifier=None, dimensions=Dimension(), system_of_units=None):
super().__init__(label, identifier)
self.fixedPoints = []
self.dimensions = dimensions
self.systemOfUnits = system_of_units
def add_fixed_point(self, fixed_point):
self.fixedPoints.append(fixed_point)
def __str__(self):
return f'{self.label()}\t<{self.identifier}> dim: {self.dimensions}'
def __eq__(self, other):
if isinstance(other, Scale):
return str(self.identifier) == str(other.identifier)
return False
def __ne__(self, other):
return not (self == other)
class RatioScale(Scale):
def __init__(self, unit, label=None, identifier=None, system_of_units=None):
if not isinstance(unit, Unit):
raise ValueError("The unit parameter in RatioScale is required to be of type Unit.")
if system_of_units is None:
system_of_units = unit.systemOfUnits
super().__init__(label, identifier, dimensions=unit.dimensions, system_of_units=system_of_units)
self.unit = unit
def base_ratio_scale(self):
return self, 0.0
def __eq__(self, other):
if isinstance(other, RatioScale):
if str(self.identifier) == str(other.identifier):
return True
if isinstance(self.identifier, URIRef) and isinstance(other.identifier, URIRef):
return False
if self.unit == other.unit:
return True
return False
def __str__(self):
return f'{self.label()}\t<{self.identifier}> unit: {self.unit} dim: {self.dimensions}'
class IntervalScale(Scale):
def __init__(self, base_scale, unit, off_set, label=None, identifier=None, system_of_units=None):
if not isinstance(unit, Unit):
raise ValueError("The unit parameter in IntervalScale is required to be of type Unit.")
if not isinstance(base_scale, Scale):
raise ValueError("The base_scale parameter in IntervalScale is required to be of type Scale.")
if unit.dimensions != base_scale.dimensions:
raise DimensionalException("The dimensions of the base scale are not the same as the dimensions of the"
" unit used. {} is not the same as {}"
.format(base_scale.dimensions, unit.dimensions))
if system_of_units is None:
system_of_units = base_scale.systemOfUnits
if system_of_units is None:
system_of_units = unit.systemOfUnits
super().__init__(label, identifier, dimensions=unit.dimensions, system_of_units=system_of_units)
self.unit = unit
self.baseScale = base_scale
self.offSet = off_set
def base_ratio_scale(self):
base_base = self.baseScale.base_ratio_scale()
conversion_fac = Unit.conversion_factor(base_base[0].unit, self.unit)
return base_base[0], base_base[1] * conversion_fac + self.offSet
def __eq__(self, other):
if isinstance(other, IntervalScale):
if str(self.identifier) == str(other.identifier):
return True
if self.unit == other.unit and self.baseScale == other.baseScale and self.offSet == other.offSet:
return True
return False
def __str__(self):
return f'{self.label()}\t<{self.identifier}> base: {self.baseScale} unit: {self.unit} dim: {self.dimensions}'
dbrepo-search-service/omlib/thing.py 0 → 100644
+ 123
− 0
View file @ 90049cef
from rdflib import BNode, URIRef, Literal, XSD
class Thing:
def __init__(self, label=None, identifier=None):
if identifier is None:
self.identifier = BNode()
else:
self.identifier = URIRef(identifier)
if label is None:
self.prefLabels = []
else:
self.prefLabels = []
self.__add_label_to_array(self.prefLabels, label, None)
self.altLabels = []
def __add_label_to_array(self, array, label, language=None):
if isinstance(label, list):
for item in label:
self.__add_label_to_array(array, item, language)
else:
if language is None:
if isinstance(label, Literal) and not label in array:
array.append(label)
else:
label_lit = Literal(label, datatype=XSD.string)
if not label_lit in array:
array.append(label_lit)
else:
if isinstance(label, Literal) and not label in array:
label_lit = Literal(label.normalize, language)
array.append(label_lit)
else:
label_lit = Literal(label, language)
if not label_lit in array:
array.append(label_lit)
def add_preferred_label(self, label, language=None):
# TODO Check if a preferred label in the specified language already exists, if so move the old one to alt labels
self.__add_label_to_array(self.prefLabels, label, language)
def add_alternative_label(self, label, language=None):
self.__add_label_to_array(self.altLabels, label, language)
def label(self):
label = self.preferred_label()
return label
def preferred_label(self, language=None):
result_label = None
if language is None:
for pref_label in self.prefLabels:
if pref_label.language is None:
result_label = pref_label
if result_label is None:
for pref_label in self.prefLabels:
if pref_label.language == 'en':
result_label = pref_label
else:
for pref_label in self.prefLabels:
if pref_label.language == language:
result_label = pref_label
return result_label
def all_labels(self):
labels = []
labels.extend(self.prefLabels)
labels.extend(self.altLabels)
return labels
class SymbolThing(Thing):
def __init__(self, name=None, symbol=None, identifier=None):
super().__init__(name, identifier)
self.dimensions = []
self.symbols = []
self.add_symbol(symbol)
def add_symbol(self, symbol, language=None):
if isinstance(symbol, list):
for item in symbol:
self.__add_label_to_array(self.symbols, item, language)
else:
if language is None:
if isinstance(symbol, Literal) and not symbol in self.symbols:
self.symbols.append(symbol)
else:
label_lit = Literal(symbol, datatype=XSD.string)
if not label_lit in self.symbols:
self.symbols.append(label_lit)
else:
if isinstance(symbol, Literal) and not symbol in self.symbols:
label_lit = Literal(symbol.normalize, language)
self.symbols.append(label_lit)
else:
label_lit = Literal(symbol, language)
if not label_lit in self.symbols:
self.symbols.append(label_lit)
def symbol(self):
symbol = self.preferred_symbol()
return symbol
def preferred_symbol(self, language=None):
result_symbol = None
if language is None:
for symbol in self.symbols:
if symbol.language is None:
result_symbol = symbol
if result_symbol is None:
for symbol in self.symbols:
if symbol.language == 'en':
result_symbol = symbol
else:
for symbol in self.symbols:
if symbol.language == language:
result_symbol = symbol
return result_symbol
def all_symbols(self):
return self.symbols
dbrepo-search-service/omlib/unit.py 0 → 100644
+ 922
− 0
View file @ 90049cef
This diff is collapsed.
dbrepo-ui/api/search.service.js
+ 9
− 6
View file @ 90049cef
...@@ -21,16 +21,19 @@ class SearchService { ...@@ -21,16 +21,19 @@ class SearchService {
search (searchData) { search (searchData) {
// transform values to what the search API expects // transform values to what the search API expects
const searchTerm = searchData.search_term let localSearchData = Object.assign({}, searchData)
delete searchData.search_term const searchTerm = localSearchData.search_term
const t1 = searchData.t1 delete localSearchData.search_term
const t2 = searchData.t2 const t1 = localSearchData.t1
searchData = Object.fromEntries(Object.entries(searchData).filter(([_, v]) => v != null && v !== '')) // https://stackoverflow.com/questions/286141/remove-blank-attributes-from-an-object-in-javascript delete localSearchData.t1
const t2 = localSearchData.t2
delete localSearchData.t2
localSearchData = Object.fromEntries(Object.entries(localSearchData).filter(([_, v]) => v != null && v !== '')) // https://stackoverflow.com/questions/286141/remove-blank-attributes-from-an-object-in-javascript
const payload = { const payload = {
t1, t1,
t2, t2,
search_term: searchTerm, search_term: searchTerm,
field_value_pairs: { ...searchData } field_value_pairs: { ...localSearchData }
} }
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
axios.post('/api/search', payload, { headers: { Accept: 'application/json' } }) axios.post('/api/search', payload, { headers: { Accept: 'application/json' } })
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment