diff --git a/dbrepo-search-db/Dockerfile b/dbrepo-search-db/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..ca9423191b7c86c8ba3c4449f0aa04167da59b78 --- /dev/null +++ b/dbrepo-search-db/Dockerfile @@ -0,0 +1,19 @@ +FROM opensearchproject/opensearch:2.8.0 as runtime + +USER root + +RUN yum install -y jq + +USER opensearch + +WORKDIR /app + +COPY ./limits.conf /etc/security/limits.conf + +COPY ./indices/*.json ./indices + +COPY ./create-indices.sh ./create-indices.sh +COPY ./docker-entrypoint.sh ./docker-entrypoint.sh +COPY ./healthcheck.sh ./healthcheck.sh + +ENTRYPOINT [ "bash", "./docker-entrypoint.sh" ] \ No newline at end of file diff --git a/dbrepo-search-db/create-indices.sh b/dbrepo-search-db/create-indices.sh new file mode 100644 index 0000000000000000000000000000000000000000..57e4ef7adbc329b58ffd6f42d777faa70174c28f --- /dev/null +++ b/dbrepo-search-db/create-indices.sh @@ -0,0 +1,16 @@ +#!/bin/bash +until curl -sSL -o /dev/null 127.0.0.1:9200/_cat/indices 2>&1 +do + echo "[create-indices.sh] OpenSearch not yet ready" + sleep 5 +done +echo "[create-indices.sh] OpenSearch ready" +for index in "user" "view" "database" "identifier" "concept" "column" "table" "unit"; do + RES=$(curl -sSL -X PUT "127.0.0.1:9200/$index" -H "Content-Type: application/json" --data "@indices/$index.json") + ACK=$(echo "$RES" | jq .acknowledged) + if [ $ACK ]; then + echo "[create-indices.sh] Created $index index" + else + echo "[create-indices.sh] Failed to create $index index: $RES" + fi +done \ No newline at end of file diff --git a/dbrepo-search-db/docker-entrypoint.sh b/dbrepo-search-db/docker-entrypoint.sh new file mode 100644 index 0000000000000000000000000000000000000000..174fd3292f3f8b6d317396e14d6b52d915156826 --- /dev/null +++ b/dbrepo-search-db/docker-entrypoint.sh @@ -0,0 +1,3 @@ +#!/bin/bash +bash /app/create-indices.sh & +bash /usr/share/opensearch/opensearch-docker-entrypoint.sh \ No newline at end of file diff --git a/dbrepo-search-db/healthcheck.sh b/dbrepo-search-db/healthcheck.sh new file mode 100644 index 0000000000000000000000000000000000000000..6f35d6cc293f7981e85cb8ba23f99026e340d82f --- /dev/null +++ b/dbrepo-search-db/healthcheck.sh @@ -0,0 +1,8 @@ +#!/bin/bash +INDICES=$(curl -sSL http://localhost:9200/_cat/indices | awk '{ if ($6) { print $3 } }') +for index in "user" "view" "database" "identifier" "concept" "column" "table" "unit"; do + if [ ! $(echo $INDICES | grep $index) ]; then + echo "[healtcheck.sh] Index $index does not exist" + exit 1 + fi +done \ No newline at end of file diff --git a/dbrepo-search-db/indices/column.json b/dbrepo-search-db/indices/column.json new file mode 100644 index 0000000000000000000000000000000000000000..f0654c92e7a4a4de579552d8f3d849905ce9f8e4 --- /dev/null +++ b/dbrepo-search-db/indices/column.json @@ -0,0 +1,112 @@ +{ + "aliases": {}, + "mappings": { + "properties": { + "auto_generated": { + "type": "boolean" + }, + "column_type": { + "type": "keyword" + }, + "concept": { + "type": "nested", + "include_in_parent": true, + "properties": { + "created": { + "type": "date", + "format": "date_optional_time||epoch_millis" + }, + "description": { + "type": "text" + }, + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "uri": { + "type": "keyword" + } + } + }, + "d": { + "type": "integer" + }, + "database_id": { + "type": "keyword" + }, + "date_format": { + "type": "nested", + "include_in_parent": true, + "properties": { + "created_at": { + "type": "date", + "format": "date_optional_time||epoch_millis" + } + } + }, + "enums": { + "type": "nested" + }, + "id": { + "type": "keyword" + }, + "index_length": { + "type": "integer" + }, + "internal_name": { + "type": "keyword" + }, + "is_null_allowed": { + "type": "boolean" + }, + "is_primary_key": { + "type": "boolean" + }, + "is_public": { + "type": "boolean" + }, + "name": { + "type": "keyword" + }, + "sets": { + "type": "nested" + }, + "size": { + "type": "integer" + }, + "table_id": { + "type": "keyword" + }, + "unit": { + "type": "nested", + "include_in_parent": true, + "properties": { + "created": { + "type": "date", + "format": "date_optional_time||epoch_millis" + }, + "description": { + "type": "text" + }, + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "uri": { + "type": "keyword" + } + } + } + } + }, + "settings": { + "index": { + "number_of_shards": "1", + "number_of_replicas": "1" + } + } +} \ No newline at end of file diff --git a/dbrepo-search-db/indices/concept.json b/dbrepo-search-db/indices/concept.json new file mode 100644 index 0000000000000000000000000000000000000000..8564e7ba7731f5f04edf7a3abc6b116ed36c2fc7 --- /dev/null +++ b/dbrepo-search-db/indices/concept.json @@ -0,0 +1,34 @@ +{ + "aliases": {}, + "mappings": { + "properties": { + "_class": { + "type": "keyword", + "index": false, + "doc_values": false + }, + "created": { + "type": "date", + "format": "date_optional_time||epoch_millis" + }, + "description": { + "type": "text" + }, + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "uri": { + "type": "keyword" + } + } + }, + "settings": { + "index": { + "number_of_shards": "1", + "number_of_replicas": "1" + } + } +} \ No newline at end of file diff --git a/dbrepo-search-db/indices/database.json b/dbrepo-search-db/indices/database.json new file mode 100644 index 0000000000000000000000000000000000000000..c363ce12fc0407e0b2ec7a93dc85c56ceb7d79c5 --- /dev/null +++ b/dbrepo-search-db/indices/database.json @@ -0,0 +1,53 @@ +{ + "aliases": {}, + "mappings": { + "properties": { + "container": { + "type": "nested", + "include_in_parent": true, + "properties": { + "created": { + "type": "date", + "format": "date_optional_time||epoch_millis" + } + } + }, + "created": { + "type": "date", + "format": "date_optional_time||epoch_millis" + }, + "creator": { + "type": "nested", + "include_in_parent": true + }, + "description": { + "type": "text" + }, + "exchange_name": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "internal_name": { + "type": "keyword" + }, + "is_public": { + "type": "boolean" + }, + "name": { + "type": "keyword" + }, + "owner": { + "type": "nested", + "include_in_parent": true + } + } + }, + "settings": { + "index": { + "number_of_shards": "1", + "number_of_replicas": "1" + } + } +} \ No newline at end of file diff --git a/dbrepo-search-db/indices/identifier.json b/dbrepo-search-db/indices/identifier.json new file mode 100644 index 0000000000000000000000000000000000000000..2d2390992ad7c9cad0220555318c0d47e998c9a6 --- /dev/null +++ b/dbrepo-search-db/indices/identifier.json @@ -0,0 +1,235 @@ +{ + "aliases": {}, + "mappings": { + "properties": { + "created": { + "type": "date", + "format": "date_optional_time||epoch_millis" + }, + "creator": { + "type": "nested", + "properties": { + "attributes": { + "type": "nested", + "include_in_parent": true, + "properties": { + "affiliation": { + "type": "keyword" + }, + "orcid": { + "type": "keyword" + } + } + }, + "firstname": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "lastname": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + }, + "creators": { + "type": "nested", + "include_in_parent": true, + "properties": { + "affiliation": { + "type": "keyword" + }, + "affiliation_identifier": { + "type": "keyword" + }, + "affiliation_identifier_scheme": { + "type": "keyword" + }, + "affiliation_identifier_scheme_uri": { + "type": "keyword" + }, + "creator_name": { + "type": "text" + }, + "firstname": { + "type": "text" + }, + "id": { + "type": "keyword" + }, + "lastname": { + "type": "text" + }, + "name_identifier": { + "type": "keyword" + }, + "name_identifier_scheme": { + "type": "keyword" + }, + "name_identifier_scheme_uri": { + "type": "keyword" + }, + "name_type": { + "type": "keyword" + } + } + }, + "database_id": { + "type": "keyword" + }, + "descriptions": { + "type": "nested", + "include_in_parent": true, + "properties": { + "description": { + "type": "text" + }, + "id": { + "type": "keyword" + }, + "language": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } + }, + "doi": { + "type": "keyword" + }, + "execution": { + "type": "date", + "format": "date_optional_time||epoch_millis" + }, + "funders": { + "type": "nested", + "include_in_parent": true, + "properties": { + "award_number": { + "type": "keyword" + }, + "award_title": { + "type": "keyword" + }, + "funder_identifier": { + "type": "keyword" + }, + "funder_identifier_type": { + "type": "keyword" + }, + "funder_name": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "scheme_uri": { + "type": "keyword" + } + } + }, + "id": { + "type": "keyword" + }, + "language": { + "type": "keyword" + }, + "licenses": { + "type": "nested", + "include_in_parent": true + }, + "publication_day": { + "type": "integer" + }, + "publication_month": { + "type": "integer" + }, + "publication_year": { + "type": "integer" + }, + "publisher": { + "type": "text" + }, + "query": { + "type": "text" + }, + "query_hash": { + "type": "text" + }, + "query_id": { + "type": "keyword" + }, + "query_normalized": { + "type": "text" + }, + "related_identifiers": { + "type": "nested", + "include_in_parent": true, + "properties": { + "created": { + "type": "date", + "format": "date_optional_time||epoch_millis" + }, + "id": { + "type": "keyword" + }, + "relation": { + "type": "keyword" + }, + "type": { + "type": "keyword" + }, + "value": { + "type": "keyword" + } + } + }, + "result_hash": { + "type": "text" + }, + "result_number": { + "type": "long" + }, + "titles": { + "type": "nested", + "include_in_parent": true, + "properties": { + "id": { + "type": "keyword" + }, + "language": { + "type": "keyword" + }, + "title": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } + }, + "type": { + "type": "keyword" + }, + "view_id": { + "type": "keyword" + }, + "visibility": { + "type": "keyword" + } + } + }, + "settings": { + "index": { + "number_of_shards": "1", + "number_of_replicas": "1" + } + } +} \ No newline at end of file diff --git a/dbrepo-search-db/indices/table.json b/dbrepo-search-db/indices/table.json new file mode 100644 index 0000000000000000000000000000000000000000..fac3d74c765aba89d4a2981f6c563ad52199b60e --- /dev/null +++ b/dbrepo-search-db/indices/table.json @@ -0,0 +1,142 @@ +{ + "aliases": {}, + "mappings": { + "properties": { + "constraints": { + "type": "nested", + "include_in_parent": true, + "properties": { + "checks": { + "type": "nested" + }, + "foreign_keys": { + "type": "nested", + "include_in_parent": true, + "properties": { + "_class": { + "type": "keyword", + "index": false, + "doc_values": false + }, + "on_delete": { + "type": "keyword" + }, + "on_update": { + "type": "keyword" + } + } + }, + "uniques": { + "type": "nested", + "include_in_parent": true, + "properties": { + "id": { + "type": "keyword" + } + } + } + } + }, + "created": { + "type": "date", + "format": "date_optional_time||epoch_millis" + }, + "creator": { + "type": "nested", + "include_in_parent": true, + "properties": { + "attributes": { + "type": "nested", + "include_in_parent": true, + "properties": { + "affiliation": { + "type": "keyword" + }, + "orcid": { + "type": "keyword" + } + } + }, + "firstname": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "lastname": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + }, + "description": { + "type": "text" + }, + "id": { + "type": "keyword" + }, + "internal_name": { + "type": "keyword" + }, + "is_public": { + "type": "boolean" + }, + "is_versioned": { + "type": "boolean" + }, + "name": { + "type": "keyword" + }, + "owner": { + "type": "nested", + "include_in_parent": true, + "properties": { + "attributes": { + "type": "nested", + "include_in_parent": true, + "properties": { + "affiliation": { + "type": "keyword" + }, + "orcid": { + "type": "keyword" + } + } + }, + "firstname": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "lastname": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + }, + "queue_name": { + "type": "keyword" + }, + "routing_key": { + "type": "keyword" + } + } + }, + "settings": { + "index": { + "number_of_shards": "1", + "number_of_replicas": "1" + } + } +} \ No newline at end of file diff --git a/dbrepo-search-db/indices/unit.json b/dbrepo-search-db/indices/unit.json new file mode 100644 index 0000000000000000000000000000000000000000..aeb66c7eaacbc7a4783b28a53b45360feb8e0981 --- /dev/null +++ b/dbrepo-search-db/indices/unit.json @@ -0,0 +1,29 @@ +{ + "aliases": {}, + "mappings": { + "properties": { + "created": { + "type": "date", + "format": "date_optional_time||epoch_millis" + }, + "description": { + "type": "text" + }, + "id": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "uri": { + "type": "keyword" + } + } + }, + "settings": { + "index": { + "number_of_shards": "1", + "number_of_replicas": "1" + } + } +} \ No newline at end of file diff --git a/dbrepo-search-db/indices/user.json b/dbrepo-search-db/indices/user.json new file mode 100644 index 0000000000000000000000000000000000000000..687518d52a514fea5ac837796907a2efc326a763 --- /dev/null +++ b/dbrepo-search-db/indices/user.json @@ -0,0 +1,40 @@ +{ + "aliases": {}, + "mappings": { + "properties": { + "attributes": { + "type": "nested", + "include_in_parent": true, + "properties": { + "affiliation": { + "type": "keyword" + }, + "orcid": { + "type": "keyword" + } + } + }, + "firstname": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "lastname": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + }, + "settings": { + "index": { + "number_of_shards": "1", + "number_of_replicas": "1" + } + } +} \ No newline at end of file diff --git a/dbrepo-search-db/indices/view.json b/dbrepo-search-db/indices/view.json new file mode 100644 index 0000000000000000000000000000000000000000..9483af5c1ea832522ff2d4c16f47f49edd2b7a03 --- /dev/null +++ b/dbrepo-search-db/indices/view.json @@ -0,0 +1,74 @@ +{ + "aliases": {}, + "mappings": { + "properties": { + "created": { + "type": "date", + "format": "date_optional_time||epoch_millis" + }, + "creator": { + "type": "nested", + "include_in_parent": true, + "properties": { + "attributes": { + "type": "nested", + "include_in_parent": true, + "properties": { + "affiliation": { + "type": "keyword" + }, + "orcid": { + "type": "keyword" + } + } + }, + "firstname": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "lastname": { + "type": "keyword" + }, + "name": { + "type": "keyword" + }, + "username": { + "type": "keyword" + } + } + }, + "database_id": { + "type": "keyword" + }, + "id": { + "type": "keyword" + }, + "initial_view": { + "type": "boolean" + }, + "internal_name": { + "type": "keyword" + }, + "is_public": { + "type": "boolean" + }, + "name": { + "type": "keyword" + }, + "query": { + "type": "text" + }, + "query_hash": { + "type": "keyword" + } + } + }, + "settings": { + "index": { + "number_of_shards": "1", + "number_of_replicas": "1" + } + } +} \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 5230afe9e7214a1d116039bf5a10966030fa0d58..44c598ea0e3e8d549de287dba23397fcfb0719c1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -212,7 +212,8 @@ services: restart: "no" container_name: dbrepo-search-db hostname: search-db - image: opensearchproject/opensearch:2.8.0 + build: ./dbrepo-search-db + image: dbrepo-search-db networks: core: ports: @@ -221,7 +222,7 @@ services: env_file: - .env healthcheck: - test: curl -s localhost:9200/_cat/indices || exit 1 + test: bash ./healthcheck.sh interval: 10s timeout: 5s retries: 12 @@ -236,7 +237,6 @@ services: memory: 4G volumes: - search-db-data:/usr/share/elasticsearch/data - - ./dbrepo-search-db/limits.conf:/etc/security/limits.conf logging: driver: json-file