Skip to content
Snippets Groups Projects
Verified Commit 61fd323b authored by Martin Weise's avatar Martin Weise
Browse files

Fixed the index generation on startup 

- Add more stability as index is generated on database creation
- Schema is fixed, any schema conflict will be detected
parent dec4e43b
No related branches found
No related tags found
4 merge requests!231CI: Remove build for log-service,!228Better error message handling in the frontend,!223Release of version 1.4.0,!192Fixed the index generation on startup
Show whitespace changes
Inline Side-by-side
Showing
with 768 additions and 3 deletions
dbrepo-search-db/Dockerfile 0 → 100644
+ 19
− 0
View file @ 61fd323b
FROM opensearchproject/opensearch:2.8.0 as runtime
USER root
RUN yum install -y jq
USER opensearch
WORKDIR /app
COPY ./limits.conf /etc/security/limits.conf
COPY ./indices/*.json ./indices
COPY ./create-indices.sh ./create-indices.sh
COPY ./docker-entrypoint.sh ./docker-entrypoint.sh
COPY ./healthcheck.sh ./healthcheck.sh
ENTRYPOINT [ "bash", "./docker-entrypoint.sh" ]
\ No newline at end of file
dbrepo-search-db/create-indices.sh 0 → 100644
+ 16
− 0
View file @ 61fd323b
#!/bin/bash
until curl -sSL -o /dev/null 127.0.0.1:9200/_cat/indices 2>&1
do
echo "[create-indices.sh] OpenSearch not yet ready"
sleep 5
done
echo "[create-indices.sh] OpenSearch ready"
for index in "user" "view" "database" "identifier" "concept" "column" "table" "unit"; do
RES=$(curl -sSL -X PUT "127.0.0.1:9200/$index" -H "Content-Type: application/json" --data "@indices/$index.json")
ACK=$(echo "$RES" | jq .acknowledged)
if [ $ACK ]; then
echo "[create-indices.sh] Created $index index"
else
echo "[create-indices.sh] Failed to create $index index: $RES"
fi
done
\ No newline at end of file
dbrepo-search-db/docker-entrypoint.sh 0 → 100644
+ 3
− 0
View file @ 61fd323b
#!/bin/bash
bash /app/create-indices.sh &
bash /usr/share/opensearch/opensearch-docker-entrypoint.sh
\ No newline at end of file
dbrepo-search-db/healthcheck.sh 0 → 100644
+ 8
− 0
View file @ 61fd323b
#!/bin/bash
INDICES=$(curl -sSL http://localhost:9200/_cat/indices | awk '{ if ($6) { print $3 } }')
for index in "user" "view" "database" "identifier" "concept" "column" "table" "unit"; do
if [ ! $(echo $INDICES | grep $index) ]; then
echo "[healtcheck.sh] Index $index does not exist"
exit 1
fi
done
\ No newline at end of file
dbrepo-search-db/indices/column.json 0 → 100644
+ 112
− 0
View file @ 61fd323b
{
"aliases": {},
"mappings": {
"properties": {
"auto_generated": {
"type": "boolean"
},
"column_type": {
"type": "keyword"
},
"concept": {
"type": "nested",
"include_in_parent": true,
"properties": {
"created": {
"type": "date",
"format": "date_optional_time||epoch_millis"
},
"description": {
"type": "text"
},
"id": {
"type": "keyword"
},
"name": {
"type": "keyword"
},
"uri": {
"type": "keyword"
}
}
},
"d": {
"type": "integer"
},
"database_id": {
"type": "keyword"
},
"date_format": {
"type": "nested",
"include_in_parent": true,
"properties": {
"created_at": {
"type": "date",
"format": "date_optional_time||epoch_millis"
}
}
},
"enums": {
"type": "nested"
},
"id": {
"type": "keyword"
},
"index_length": {
"type": "integer"
},
"internal_name": {
"type": "keyword"
},
"is_null_allowed": {
"type": "boolean"
},
"is_primary_key": {
"type": "boolean"
},
"is_public": {
"type": "boolean"
},
"name": {
"type": "keyword"
},
"sets": {
"type": "nested"
},
"size": {
"type": "integer"
},
"table_id": {
"type": "keyword"
},
"unit": {
"type": "nested",
"include_in_parent": true,
"properties": {
"created": {
"type": "date",
"format": "date_optional_time||epoch_millis"
},
"description": {
"type": "text"
},
"id": {
"type": "keyword"
},
"name": {
"type": "keyword"
},
"uri": {
"type": "keyword"
}
}
}
}
},
"settings": {
"index": {
"number_of_shards": "1",
"number_of_replicas": "1"
}
}
}
\ No newline at end of file
dbrepo-search-db/indices/concept.json 0 → 100644
+ 34
− 0
View file @ 61fd323b
{
"aliases": {},
"mappings": {
"properties": {
"_class": {
"type": "keyword",
"index": false,
"doc_values": false
},
"created": {
"type": "date",
"format": "date_optional_time||epoch_millis"
},
"description": {
"type": "text"
},
"id": {
"type": "keyword"
},
"name": {
"type": "keyword"
},
"uri": {
"type": "keyword"
}
}
},
"settings": {
"index": {
"number_of_shards": "1",
"number_of_replicas": "1"
}
}
}
\ No newline at end of file
dbrepo-search-db/indices/database.json 0 → 100644
+ 53
− 0
View file @ 61fd323b
{
"aliases": {},
"mappings": {
"properties": {
"container": {
"type": "nested",
"include_in_parent": true,
"properties": {
"created": {
"type": "date",
"format": "date_optional_time||epoch_millis"
}
}
},
"created": {
"type": "date",
"format": "date_optional_time||epoch_millis"
},
"creator": {
"type": "nested",
"include_in_parent": true
},
"description": {
"type": "text"
},
"exchange_name": {
"type": "keyword"
},
"id": {
"type": "keyword"
},
"internal_name": {
"type": "keyword"
},
"is_public": {
"type": "boolean"
},
"name": {
"type": "keyword"
},
"owner": {
"type": "nested",
"include_in_parent": true
}
}
},
"settings": {
"index": {
"number_of_shards": "1",
"number_of_replicas": "1"
}
}
}
\ No newline at end of file
dbrepo-search-db/indices/identifier.json 0 → 100644
+ 235
− 0
View file @ 61fd323b
{
"aliases": {},
"mappings": {
"properties": {
"created": {
"type": "date",
"format": "date_optional_time||epoch_millis"
},
"creator": {
"type": "nested",
"properties": {
"attributes": {
"type": "nested",
"include_in_parent": true,
"properties": {
"affiliation": {
"type": "keyword"
},
"orcid": {
"type": "keyword"
}
}
},
"firstname": {
"type": "keyword"
},
"id": {
"type": "keyword"
},
"lastname": {
"type": "keyword"
},
"name": {
"type": "keyword"
},
"username": {
"type": "keyword"
}
}
},
"creators": {
"type": "nested",
"include_in_parent": true,
"properties": {
"affiliation": {
"type": "keyword"
},
"affiliation_identifier": {
"type": "keyword"
},
"affiliation_identifier_scheme": {
"type": "keyword"
},
"affiliation_identifier_scheme_uri": {
"type": "keyword"
},
"creator_name": {
"type": "text"
},
"firstname": {
"type": "text"
},
"id": {
"type": "keyword"
},
"lastname": {
"type": "text"
},
"name_identifier": {
"type": "keyword"
},
"name_identifier_scheme": {
"type": "keyword"
},
"name_identifier_scheme_uri": {
"type": "keyword"
},
"name_type": {
"type": "keyword"
}
}
},
"database_id": {
"type": "keyword"
},
"descriptions": {
"type": "nested",
"include_in_parent": true,
"properties": {
"description": {
"type": "text"
},
"id": {
"type": "keyword"
},
"language": {
"type": "keyword"
},
"type": {
"type": "keyword"
}
}
},
"doi": {
"type": "keyword"
},
"execution": {
"type": "date",
"format": "date_optional_time||epoch_millis"
},
"funders": {
"type": "nested",
"include_in_parent": true,
"properties": {
"award_number": {
"type": "keyword"
},
"award_title": {
"type": "keyword"
},
"funder_identifier": {
"type": "keyword"
},
"funder_identifier_type": {
"type": "keyword"
},
"funder_name": {
"type": "keyword"
},
"id": {
"type": "keyword"
},
"scheme_uri": {
"type": "keyword"
}
}
},
"id": {
"type": "keyword"
},
"language": {
"type": "keyword"
},
"licenses": {
"type": "nested",
"include_in_parent": true
},
"publication_day": {
"type": "integer"
},
"publication_month": {
"type": "integer"
},
"publication_year": {
"type": "integer"
},
"publisher": {
"type": "text"
},
"query": {
"type": "text"
},
"query_hash": {
"type": "text"
},
"query_id": {
"type": "keyword"
},
"query_normalized": {
"type": "text"
},
"related_identifiers": {
"type": "nested",
"include_in_parent": true,
"properties": {
"created": {
"type": "date",
"format": "date_optional_time||epoch_millis"
},
"id": {
"type": "keyword"
},
"relation": {
"type": "keyword"
},
"type": {
"type": "keyword"
},
"value": {
"type": "keyword"
}
}
},
"result_hash": {
"type": "text"
},
"result_number": {
"type": "long"
},
"titles": {
"type": "nested",
"include_in_parent": true,
"properties": {
"id": {
"type": "keyword"
},
"language": {
"type": "keyword"
},
"title": {
"type": "keyword"
},
"type": {
"type": "keyword"
}
}
},
"type": {
"type": "keyword"
},
"view_id": {
"type": "keyword"
},
"visibility": {
"type": "keyword"
}
}
},
"settings": {
"index": {
"number_of_shards": "1",
"number_of_replicas": "1"
}
}
}
\ No newline at end of file
dbrepo-search-db/indices/table.json 0 → 100644
+ 142
− 0
View file @ 61fd323b
{
"aliases": {},
"mappings": {
"properties": {
"constraints": {
"type": "nested",
"include_in_parent": true,
"properties": {
"checks": {
"type": "nested"
},
"foreign_keys": {
"type": "nested",
"include_in_parent": true,
"properties": {
"_class": {
"type": "keyword",
"index": false,
"doc_values": false
},
"on_delete": {
"type": "keyword"
},
"on_update": {
"type": "keyword"
}
}
},
"uniques": {
"type": "nested",
"include_in_parent": true,
"properties": {
"id": {
"type": "keyword"
}
}
}
}
},
"created": {
"type": "date",
"format": "date_optional_time||epoch_millis"
},
"creator": {
"type": "nested",
"include_in_parent": true,
"properties": {
"attributes": {
"type": "nested",
"include_in_parent": true,
"properties": {
"affiliation": {
"type": "keyword"
},
"orcid": {
"type": "keyword"
}
}
},
"firstname": {
"type": "keyword"
},
"id": {
"type": "keyword"
},
"lastname": {
"type": "keyword"
},
"name": {
"type": "keyword"
},
"username": {
"type": "keyword"
}
}
},
"description": {
"type": "text"
},
"id": {
"type": "keyword"
},
"internal_name": {
"type": "keyword"
},
"is_public": {
"type": "boolean"
},
"is_versioned": {
"type": "boolean"
},
"name": {
"type": "keyword"
},
"owner": {
"type": "nested",
"include_in_parent": true,
"properties": {
"attributes": {
"type": "nested",
"include_in_parent": true,
"properties": {
"affiliation": {
"type": "keyword"
},
"orcid": {
"type": "keyword"
}
}
},
"firstname": {
"type": "keyword"
},
"id": {
"type": "keyword"
},
"lastname": {
"type": "keyword"
},
"name": {
"type": "keyword"
},
"username": {
"type": "keyword"
}
}
},
"queue_name": {
"type": "keyword"
},
"routing_key": {
"type": "keyword"
}
}
},
"settings": {
"index": {
"number_of_shards": "1",
"number_of_replicas": "1"
}
}
}
\ No newline at end of file
dbrepo-search-db/indices/unit.json 0 → 100644
+ 29
− 0
View file @ 61fd323b
{
"aliases": {},
"mappings": {
"properties": {
"created": {
"type": "date",
"format": "date_optional_time||epoch_millis"
},
"description": {
"type": "text"
},
"id": {
"type": "keyword"
},
"name": {
"type": "keyword"
},
"uri": {
"type": "keyword"
}
}
},
"settings": {
"index": {
"number_of_shards": "1",
"number_of_replicas": "1"
}
}
}
\ No newline at end of file
dbrepo-search-db/indices/user.json 0 → 100644
+ 40
− 0
View file @ 61fd323b
{
"aliases": {},
"mappings": {
"properties": {
"attributes": {
"type": "nested",
"include_in_parent": true,
"properties": {
"affiliation": {
"type": "keyword"
},
"orcid": {
"type": "keyword"
}
}
},
"firstname": {
"type": "keyword"
},
"id": {
"type": "keyword"
},
"lastname": {
"type": "keyword"
},
"name": {
"type": "keyword"
},
"username": {
"type": "keyword"
}
}
},
"settings": {
"index": {
"number_of_shards": "1",
"number_of_replicas": "1"
}
}
}
\ No newline at end of file
dbrepo-search-db/indices/view.json 0 → 100644
+ 74
− 0
View file @ 61fd323b
{
"aliases": {},
"mappings": {
"properties": {
"created": {
"type": "date",
"format": "date_optional_time||epoch_millis"
},
"creator": {
"type": "nested",
"include_in_parent": true,
"properties": {
"attributes": {
"type": "nested",
"include_in_parent": true,
"properties": {
"affiliation": {
"type": "keyword"
},
"orcid": {
"type": "keyword"
}
}
},
"firstname": {
"type": "keyword"
},
"id": {
"type": "keyword"
},
"lastname": {
"type": "keyword"
},
"name": {
"type": "keyword"
},
"username": {
"type": "keyword"
}
}
},
"database_id": {
"type": "keyword"
},
"id": {
"type": "keyword"
},
"initial_view": {
"type": "boolean"
},
"internal_name": {
"type": "keyword"
},
"is_public": {
"type": "boolean"
},
"name": {
"type": "keyword"
},
"query": {
"type": "text"
},
"query_hash": {
"type": "keyword"
}
}
},
"settings": {
"index": {
"number_of_shards": "1",
"number_of_replicas": "1"
}
}
}
\ No newline at end of file
docker-compose.yml
+ 3
− 3
View file @ 61fd323b
......@@ -212,7 +212,8 @@ services:
restart: "no"
container_name: dbrepo-search-db
hostname: search-db
image: opensearchproject/opensearch:2.8.0
build: ./dbrepo-search-db
image: dbrepo-search-db
networks:
core:
ports:
......@@ -221,7 +222,7 @@ services:
env_file:
- .env
healthcheck:
test: curl -s localhost:9200/_cat/indices || exit 1
test: bash ./healthcheck.sh
interval: 10s
timeout: 5s
retries: 12
......@@ -236,7 +237,6 @@ services:
memory: 4G
volumes:
- search-db-data:/usr/share/elasticsearch/data
- ./dbrepo-search-db/limits.conf:/etc/security/limits.conf
logging:
driver: json-file
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment