diff --git a/fda-analyse-service/app.py b/fda-analyse-service/app.py index 12213a9b4fc7eef683e03c1e247335dcad816e53..033dc8e16503a39515a6aede9ec16a21dbd915c0 100644 --- a/fda-analyse-service/app.py +++ b/fda-analyse-service/app.py @@ -130,9 +130,9 @@ def mdb_basicstatistic(): try: dbid = int(input_json['dbid']) tid = int(input_json['tid']) - res = update_bstatistic(dbid, tid) + update_bstatistic(dbid, tid) logging.info('Update metadata database entity mdb_columns_cat, mdb_columns_num, mdb_columns_nom') - return jsonify(res), 200 + return "OK", 200 except Exception as e: logging.error(e) res = {"success": False, "message": "Unknown error"} diff --git a/logstash/pipeline/logstash-1.conf b/logstash/pipeline/logstash-1.conf new file mode 100644 index 0000000000000000000000000000000000000000..0679a2a84b62d9dcfbd437cccb9ad3de1b6a2c53 --- /dev/null +++ b/logstash/pipeline/logstash-1.conf @@ -0,0 +1,24 @@ +input { + jdbc { + jdbc_connection_string => "jdbc:postgresql://fda-metadata-db:5432/fda" + jdbc_user => "postgres" + jdbc_password => "postgres" + jdbc_driver_class => "org.postgresql.Driver" + jdbc_driver_library => "/usr/share/logstash/postgresql.jar" + statement => "SELECT cdbid, tid, cid, num_cat, cat_array, last_modified, created from mdb_columns_cat;" + schedule => "*/1 * * * *" + } +} + output { + stdout { } + } + output { + elasticsearch { + hosts => ["http://fda-search-service:9200"] + index => "columnstypeindex" + document_id => "%{cdbid}.%{tid}.%{cid}" + doc_as_upsert => true + #user => "es_user" + password => "elastic" + } +} \ No newline at end of file diff --git a/logstash/pipeline/logstash-2.conf b/logstash/pipeline/logstash-2.conf new file mode 100644 index 0000000000000000000000000000000000000000..e7b5360cb3da03dfb24fe6ea6715179e29cb95d2 --- /dev/null +++ b/logstash/pipeline/logstash-2.conf @@ -0,0 +1,24 @@ +input { + jdbc { + jdbc_connection_string => "jdbc:postgresql://fda-metadata-db:5432/fda" + jdbc_user => "postgres" + jdbc_password => "postgres" + jdbc_driver_class => "org.postgresql.Driver" + jdbc_driver_library => "/usr/share/logstash/postgresql.jar" + statement => "SELECT cdbid, tid, cid, maxlength, last_modified, created from mdb_columns_nom;" + schedule => "*/1 * * * *" + } +} +output { + stdout { } + } +output { + elasticsearch { + hosts => ["http://fda-search-service:9200"] + index => "columnstypeindex" + document_id => "%{cdbid}.%{tid}.%{cid}" + doc_as_upsert => true + #user => "es_user" + password => "elastic" + } +} \ No newline at end of file diff --git a/logstash/pipeline/logstash-3.conf b/logstash/pipeline/logstash-3.conf new file mode 100644 index 0000000000000000000000000000000000000000..598c8471dd9243ac4d76fcdb427df094df3bcf86 --- /dev/null +++ b/logstash/pipeline/logstash-3.conf @@ -0,0 +1,30 @@ +input { + jdbc { + jdbc_connection_string => "jdbc:postgresql://fda-metadata-db:5432/fda" + jdbc_user => "postgres" + jdbc_password => "postgres" + jdbc_driver_class => "org.postgresql.Driver" + jdbc_driver_library => "/usr/share/logstash/postgresql.jar" + statement => "SELECT cdbid, tid, cid, siunit, maxval, minval, mean, median, sd, last_modified, created from mdb_columns_num;" + schedule => "*/1 * * * *" + } +} +# filter { +# json { +# source => "message" +# remove_field => [ "histogram" ] +# } +# } +output { + stdout { } + } +output { + elasticsearch { + hosts => ["http://fda-search-service:9200"] + index => "columnstypeindex" + document_id => "%{cdbid}.%{tid}.%{cid}" + doc_as_upsert => true + #user => "es_user" + password => "elastic" + } +} \ No newline at end of file diff --git a/logstash/pipeline/logstash.conf b/logstash/pipeline/logstash.conf deleted file mode 100644 index ad956c76c000dce823aa2d0f86572c003f1a0185..0000000000000000000000000000000000000000 --- a/logstash/pipeline/logstash.conf +++ /dev/null @@ -1,60 +0,0 @@ -input { - jdbc { - jdbc_connection_string => "jdbc:postgresql://fda-metadata-db:5432/fda" - jdbc_user => "postgres" - jdbc_password => "postgres" - jdbc_driver_class => "org.postgresql.Driver" - jdbc_driver_library => "/usr/share/logstash/postgresql.jar" - statement => "SELECT * from mdb_columns_cat;" - schedule => "*/1 * * * *" - } -} output { - elasticsearch { - hosts => ["http://fda-search-service:9200"] - index => "columntypeindex" - document_id => "mdb_columns_cat" - doc_as_upsert => true - #user => "es_user" - password => "elastic" - } -} -input { - jdbc { - jdbc_connection_string => "jdbc:postgresql://fda-metadata-db:5432/fda" - jdbc_user => "postgres" - jdbc_password => "postgres" - jdbc_driver_class => "org.postgresql.Driver" - jdbc_driver_library => "/usr/share/logstash/postgresql.jar" - statement => "SELECT * from mdb_columns_nom;" - schedule => "*/1 * * * *" - } -} output { - elasticsearch { - hosts => ["http://fda-search-service:9200"] - index => "columntypeindex" - document_id => "mdb_columns_nom" - doc_as_upsert => true - #user => "es_user" - password => "elastic" - } -} -input { - jdbc { - jdbc_connection_string => "jdbc:postgresql://fda-metadata-db:5432/fda" - jdbc_user => "postgres" - jdbc_password => "postgres" - jdbc_driver_class => "org.postgresql.Driver" - jdbc_driver_library => "/usr/share/logstash/postgresql.jar" - statement => "SELECT * from mdb_columns_num;" - schedule => "*/1 * * * *" - } -} output { - elasticsearch { - hosts => ["http://fda-search-service:9200"] - index => "columntypeindex" - document_id => "mdb_columns_num" - doc_as_upsert => true - #user => "es_user" - password => "elastic" - } -} \ No newline at end of file