WIP

Signed-off-by: Martin Weise <martin.weise@tuwien.ac.at>

dbrepo-analyse-service/Dockerfile

-FROM --platform=$BUILDPLATFORM python:3.11-alpine3.21
+FROM --platform=$BUILDPLATFORM pytorch/pytorch:2.7.0-cuda12.6-cudnn9-runtime
 LABEL org.opencontainers.image.authors="martin.weise@tuwien.ac.at"
 
-RUN apk --no-cache add \
+RUN apt-get update &&  \
+    apt-get install -y \
       bash \
-    curl \
-    openjdk8-jre
+      curl && \
+    apt-get clean
 
 COPY Pipfile Pipfile.lock ./
 
@@ -14,7 +15,7 @@ RUN pip install pipenv && \
     pipenv install gunicorn && \
     pipenv install --system --deploy
 
-RUN adduser -D dbrepo --uid 1001
+RUN adduser dbrepo --uid 1001
 
 WORKDIR /app
 

dbrepo-analyse-service/Pipfile

@@ -26,13 +26,13 @@ opensearch-py = "*"
 ecs_logging = "*"
 flask_caching = "*"
 owlready2 = "*"
-torch = "*"
 sentence_transformers = "*"
 FlagEmbedding = "*"
-transformers = "4.44.2"
+transformers = "==4.44.2"
 textdistance = "*"
 h5py = "*"
 nltk = "*"
+matplotlib = "*"
 
 [dev-packages]
 coverage = "*"

dbrepo-analyse-service/app.py

@@ -245,9 +245,8 @@ app.config["SPARK_ENDPOINT"] = os.getenv('SPARK_ENDPOINT', 'local[2]')
 app.config["METADATA_SERVICE_ENDPOINT"] = os.getenv('METADATA_SERVICE_ENDPOINT', 'http://localhost')
 app.config["SYSTEM_USERNAME"] = os.getenv('SYSTEM_USERNAME', 'admin')
 app.config["SYSTEM_PASSWORD"] = os.getenv('SYSTEM_PASSWORD', 'admin')
-app.config["OBJECT_SCORING_METHOD"] = os.getenv('OBJECT_SCORING_METHOD', 'cos1')
-app.config["UNIT_SCORING_METHOD"] = os.getenv('UNIT_SCORING_METHOD', 'cos2')
 app.config["CACHE_DIR"] = os.getenv('CACHE_DIR', './cache')
+app.config["CACHE_TYPE"] = os.getenv('CACHE_TYPE', 'filesystem')
 
 initialize_service.initialize_cache(cache, app.config["CACHE_DIR"])
 

dbrepo-analyse-service/services/initialize_service.py

-from flask import current_app
-
 import services.load_service as load_service
 from matching.embeddings import BGEMDense
 from matching.scoring import Levenshtein, CosineSimilarity, JaroWinkler, Jaccard, Lcsseq
@@ -30,8 +28,7 @@ def load_all():
     set_unit_embeddings(load_service._unit_ontology_files_information, are_units_created, unit_ontologies)
 
     # Match Service
-    init_match_service(scoring_methods, current_app.config["OBJECT_SCORING_METHOD"],
-                       current_app.config["UNIT_SCORING_METHOD"])
+    init_match_service(scoring_methods, "cos1", "cos2")
 
 
 def initialize_cache(cache, cache_dir):

dbrepo-gateway-service/dbrepo.conf

@@ -109,6 +109,15 @@ server {
         proxy_read_timeout      90;
     }
 
+    location /api/ontology {
+        proxy_set_header        X-Real-IP $remote_addr;
+        proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
+        proxy_set_header        X-Forwarded-Host $host;
+        proxy_set_header        X-Forwarded-Proto $scheme;
+        proxy_pass              http://analyse-service:8080;
+        proxy_read_timeout      90;
+    }
+
     location ~ "/api/database/([0-9a-f]{8}-[0-9a-f]{4}-[4][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})/table/([0-9a-f]{8}-[0-9a-f]{4}-[4][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12})/(data|statistic|history)" {
         proxy_set_header        X-Real-IP $remote_addr;
         proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
@@ -145,7 +154,7 @@ server {
         proxy_read_timeout      600;
     }
 
-    location ~ "/api/(database|concept|container|identifier|image|message|license|oai|ontology|unit|user)" {
+    location ~ "/api/(database|concept|container|identifier|image|message|license|oai|unit|user)" {
         proxy_set_header        X-Real-IP $remote_addr;
         proxy_set_header        X-Forwarded-For $proxy_add_x_forwarded_for;
         proxy_set_header        X-Forwarded-Host $host;

dbrepo-ui-new/requirements.txt

-aiohappyeyeballs==2.4.4
-aiohttp==3.11.11
-aiosignal==1.3.2
-annotated-types==0.7.0
-attrs==24.3.0
-blinker==1.8.2
-certifi==2024.12.14
-charset-normalizer==3.4.1
-click==8.1.7
-colorama==0.4.6
-dbrepo==1.6.0
-Flask==3.0.3
-frozenlist==1.5.0
-idna==3.10
-itsdangerous==2.2.0
-Jinja2==3.1.4
-MarkupSafe==2.1.5
-multidict==6.1.0
-numpy==2.2.1
-pandas==2.2.3
-pika==1.3.2
-propcache==0.2.1
-pydantic==2.10.5
-pydantic_core==2.27.2
-python-dateutil==2.9.0.post0
-python-dotenv==1.0.1
-pytz==2024.2
-requests==2.32.3
-six==1.17.0
-tinydb==4.8.2
-tuspy==1.1.0
-typing_extensions==4.12.2
-tzdata==2024.2
-urllib3==2.3.0
-watchdog==5.0.2
-Werkzeug==3.0.4
-yarl==1.18.3

dbrepo-ui/Dockerfile

@@ -10,6 +10,7 @@ RUN bun install
 
 ENV NODE_ENV="production"
 
+COPY ./api ./api
 COPY ./assets ./assets
 COPY ./components ./components
 COPY ./composables ./composables

dbrepo-ui/api/so.api.service.js

+export default class SOApiService {
+  async post_matcher(url_method, _body) {
+    const requestOptions = {
+      method: 'POST',
+      headers: {
+        Accept: 'application/json',
+        'Content-Type': 'application/json',
+        'Access-Control-Allow-Origin': '*'
+      },
+      body: JSON.stringify(_body),
+    };
+    const config = useRuntimeConfig()
+    return await fetch(`${config.public.api.client}/api/ontology/${url_method}`, requestOptions)
+      .then((response) => {
+        return response;
+
+      }).catch((error) => {
+        console.log('post_matcher Error:', error);
+        return error;
+      });
+  }
+
+  async get_matcher(url_method, schema_id, source_filter, target_filter) {
+    const requestOptions = {
+      method: 'GET',
+      headers: {
+        Accept: 'application/json',
+        'Content-Type': 'application/json',
+        'Access-Control-Allow-Origin': '*'
+      },
+    };
+    const config = useRuntimeConfig()
+    return await fetch(`${config.public.api.client}/api/ontology/${url_method}?schema_id=${schema_id}&source_filter=${source_filter}&target_filter=${target_filter}`, requestOptions)
+      .then((response) => {
+        return response;
+
+      }).catch((error) => {
+        console.log('get_matcher Error:', error);
+        return error;
+      });
+  }
+}

dbrepo-ui/api/so.extractor.service.js

+export default class SOExtractorService {
+    extract_ontology_header_data(ontology_data) {
+        var data = [];
+  
+        for (let i = 0; i < ontology_data["data_ontology_names"].length; i++) {
+            data.push({guid: ontology_data["data_ontology_names"][i], id: ontology_data["data_ontology_names"][i], text: ":"+ontology_data["data_ontology_names"][i] + "  " + ontology_data["data_ontology_means"][i] + " +/- " + ontology_data["data_ontology_stds"][i]});
+        }
+        return data;
+    }
+
+    extract_ontology_winners_data_object(object_ontology, column_name){
+        var data = [];
+        var targets = object_ontology["data_ontology_winner_targets"][column_name];
+        var ontologies = object_ontology["data_ontology_winner_ontologies"][column_name];
+        var scores = object_ontology["data_ontology_winner_scores"][column_name];
+        var constraints = object_ontology["data_ontology_winner_constraint_bools"][column_name];
+  
+        for (let i = 0; i < targets.length; i++) {
+          var displayedName = targets[i] + "  " + scores[i] + " " + constraints[i]+ " :" + ontologies[i];
+          data.push({guid: targets[i]+ontologies[i], id: targets[i], text: displayedName, ontology: ontologies[i], score: scores[i], constraint: constraints[i]});
+        }
+        return this.addNoneColumn(data);
+    }
+
+    extract_ontology_winners_data_unit(unit_ontology, column_name){
+        var data = [];
+        var targets = unit_ontology["data_ontology_winner_targets"][column_name];
+        var ontologies = unit_ontology["data_ontology_winner_ontologies"][column_name];
+        var scores = unit_ontology["data_ontology_winner_scores"][column_name];
+        for (let i = 0; i < targets.length; i++) {
+          var displayedName = targets[i] + "  " + scores[i] + " :" + ontologies[i];
+          data.push({guid: targets[i]+ontologies[i], id: targets[i], text: displayedName, ontology: ontologies[i], score: scores[i], constraint: true});
+        }
+        return this.addNoneColumn(data);
+    }
+
+    addNoneColumn(data){
+        data.push({guid: "none", id: "none", text: "none", isDummy: true});
+        return data;
+    }
+
+    // Suggestions
+    extract_winner_ontology(so, type, ontology_data){
+        if(so.getHeaderOntologySelectionSuggested(type) == false) return so.getHeaderOntologySelection(type); // User selected by hand. No change.
+        so.setHeaderOntologySelection(type, ontology_data["data_ontology_names"][0], true);
+        return ontology_data["data_ontology_names"][0];
+    }
+
+    extract_winner_column(so, type, ontology_data, column_name){
+        if(so.getColumnSelectionSuggested(type, column_name) == false) return so.getColumnSelection(type, column_name); // User selected by hand. No change.
+        var targets = ontology_data["data_ontology_winner_targets"][column_name];
+        var ontologies = ontology_data["data_ontology_winner_ontologies"][column_name];
+        for (let i = 0; i < targets.length; i++) {
+            if (so.headerOntologySelections[type]?.suggested && ontologies[i] == ontology_data["data_ontology_names"][0]){ // User has no selection and Target is in the winner ontology
+                so.setColumnSelection(type, column_name, targets[i], ontologies[i], true);
+                if(i == 0){ // Target is the winner target
+                    return targets[i]+" "; //" " added to disable autoselect enforcement of v-combobox
+                };
+                return targets[i]+"*"; // Target is not the winner target
+            }
+            else if(!so.headerOntologySelections[type]?.suggested && ontologies[i] == so.headerOntologySelections[type].name){ // User has selection and Target is in the selected ontology
+                so.setColumnSelection(type, column_name, targets[i], ontologies[i], true);
+                if(i == 0){ // Target is the winner target
+                    return targets[i]+" ";
+                };
+                return targets[i]+"*"; // Target is not the winner target
+            }
+        }
+    }
+}
\ No newline at end of file

dbrepo-ui/components/table/TableSchema.vue

@@ -15,6 +15,38 @@
           </v-alert>
         </v-col>
       </v-row>
+      <v-row>
+        <v-col
+          lg="8">
+          <v-combobox
+            disable-sort
+            label="Object Ontology"
+            :value="extractor.extract_winner_ontology(so, 'object', so.object_ontology)"
+            v-if="getIsLoaded.object_header == true"
+            :items="extractor.extract_ontology_header_data(so.object_ontology)"
+            :style="{'font-weight': `${so.headerOntologySelections.object?.suggested == true ? 'normal' : 'bold'}`}"
+            item-value="guid"
+            item-text="text"
+            v-on:change="so.ontologyInteraction('object', $event)"
+            variant="underlined" />
+        </v-col>
+      </v-row>
+      <v-row>
+        <v-col
+          lg="8">
+          <v-combobox
+            disable-sort
+            label="Unit Ontology"
+            :value="extractor.extract_winner_ontology(so, 'unit', so.unit_ontology)"
+            v-if="getIsLoaded.unit_header == true"
+            :items="extractor.extract_ontology_header_data(so.unit_ontology)"
+            :style="{'font-weight': `${so.headerOntologySelections.unit?.suggested == true ? 'normal' : 'bold'}`}"
+            item-value="guid"
+            item-text="text"
+            v-on:change="so.ontologyInteraction('unit', $event)"
+            variant="underlined" />
+        </v-col>
+      </v-row>
       <v-row
         v-for="(c, idx) in columns"
         :key="`r-${idx}`"
@@ -198,6 +230,8 @@
 
 <script>
 import { useCacheStore } from '@/stores/cache.js'
+import SOExtractorService from '@/api/so.extractor.service'
+import SOService from '@/api/so.service'
 
 export default {
   props: {
@@ -228,6 +262,10 @@ export default {
   },
   data () {
     return {
+      schema_id: null,
+      extractor: new SOExtractorService(),
+      so: new SOService(),
+      finished: false,
       valid: false,
       columnTypes: [],
       loadColumn: false,
@@ -253,6 +291,12 @@ export default {
         return false
       }
       return this.columns.filter(c => c.primary_key).length === 0
+    },
+    getColumns(){
+      return Object.assign({}, this.columns);
+    },
+    getIsLoaded(){
+      return Object.assign({}, this.so.isLoaded);
     }
   },
   watch: {
@@ -260,6 +304,29 @@ export default {
       handler () {
         this.$emit('schema-valid', { valid: this.valid })
       }
+    },
+    getColumns: {
+      deep: true,
+      handler (newValue, oldValue) {
+        if (Object.keys(oldValue).length == 0 && Object.keys(newValue).length > 0) { // initial
+          var column_names = [];
+          var column_datatypes = {};
+          for (let i = 0; i < this.columns.length; i++) {
+            const c = this.columns[i];
+            column_names.push(c.name);
+            column_datatypes[c.name] = c.type;
+          }
+          this.so.init(column_names, column_datatypes);
+        } else if (Object.keys(oldValue).length-Object.keys(newValue).length == 1) { // remove column
+          var removed_column = null;  // find the removed column
+          oldValue = Object.values(oldValue).map(value => value.name);
+          newValue = Object.values(newValue).map(value => value.name);
+          var removed_column = oldValue.filter(x => !newValue.includes(x));
+          if(removed_column.length > 0) this.so.deleteColumn(removed_column[0]); // Only one column can be removed at a time
+        }
+        else if (Object.keys(oldValue).length-Object.keys(newValue).length == -1) {} // add column: implicitly done in changeSourceName and changeSourceDatatype
+        else {}
+      }
     }
   },
   mounted () {