exclude histogram from columntypeindex, modify columntypeindex

fda-analyse-service/app.py

@@ -130,9 +130,9 @@ def mdb_basicstatistic():
     try:
         dbid = int(input_json['dbid'])
         tid = int(input_json['tid'])
-        res = update_bstatistic(dbid, tid)
+        update_bstatistic(dbid, tid)
         logging.info('Update metadata database entity mdb_columns_cat, mdb_columns_num, mdb_columns_nom')
-        return jsonify(res), 200
+        return "OK", 200
     except Exception as e:
         logging.error(e)
         res = {"success": False, "message": "Unknown error"}

logstash/pipeline/logstash-1.conf

+input {
+  jdbc {
+     jdbc_connection_string => "jdbc:postgresql://fda-metadata-db:5432/fda"
+     jdbc_user => "postgres"
+     jdbc_password => "postgres"
+     jdbc_driver_class => "org.postgresql.Driver"
+     jdbc_driver_library => "/usr/share/logstash/postgresql.jar"
+     statement => "SELECT cdbid, tid, cid, num_cat, cat_array, last_modified, created from mdb_columns_cat;"
+     schedule => "*/1 * * * *"
+ }
+}
+ output {
+      stdout { }
+    }
+    output {
+  elasticsearch {
+    hosts => ["http://fda-search-service:9200"]
+    index => "columnstypeindex"
+    document_id => "%{cdbid}.%{tid}.%{cid}"
+    doc_as_upsert => true
+    #user => "es_user"
+    password => "elastic"
+ }
+}
\ No newline at end of file

logstash/pipeline/logstash-2.conf

+input {
+  jdbc {
+     jdbc_connection_string => "jdbc:postgresql://fda-metadata-db:5432/fda"
+     jdbc_user => "postgres"
+     jdbc_password => "postgres"
+     jdbc_driver_class => "org.postgresql.Driver"
+     jdbc_driver_library => "/usr/share/logstash/postgresql.jar"
+     statement => "SELECT  cdbid, tid, cid, maxlength, last_modified, created from mdb_columns_nom;"
+     schedule => "*/1 * * * *"
+ }
+}
+output {
+      stdout { }
+    }
+output {
+  elasticsearch {
+    hosts => ["http://fda-search-service:9200"]
+    index => "columnstypeindex"
+    document_id => "%{cdbid}.%{tid}.%{cid}"
+    doc_as_upsert => true
+    #user => "es_user"
+    password => "elastic"
+ }
+}
\ No newline at end of file

logstash/pipeline/logstash-3.conf

+input {
+  jdbc {
+     jdbc_connection_string => "jdbc:postgresql://fda-metadata-db:5432/fda"
+     jdbc_user => "postgres"
+     jdbc_password => "postgres"
+     jdbc_driver_class => "org.postgresql.Driver"
+     jdbc_driver_library => "/usr/share/logstash/postgresql.jar"
+     statement => "SELECT cdbid, tid, cid, siunit, maxval, minval, mean, median, sd, last_modified, created from mdb_columns_num;"
+     schedule => "*/1 * * * *"
+ }
+}
+# filter {
+#       json {
+#         source => "message"
+#         remove_field => [ "histogram" ]
+#       }
+#     }
+output {
+      stdout { }
+    }
+output {
+  elasticsearch {
+    hosts => ["http://fda-search-service:9200"]
+    index => "columnstypeindex"
+    document_id => "%{cdbid}.%{tid}.%{cid}"
+    doc_as_upsert => true
+    #user => "es_user"
+    password => "elastic"
+ }
+}
\ No newline at end of file

logstash/pipeline/logstash.conf

-input {
-  jdbc {
-     jdbc_connection_string => "jdbc:postgresql://fda-metadata-db:5432/fda"
-     jdbc_user => "postgres"
-     jdbc_password => "postgres"
-     jdbc_driver_class => "org.postgresql.Driver"
-     jdbc_driver_library => "/usr/share/logstash/postgresql.jar"
-     statement => "SELECT * from mdb_columns_cat;"
-     schedule => "*/1 * * * *"
- }
-} output {
-  elasticsearch {
-    hosts => ["http://fda-search-service:9200"]
-    index => "columntypeindex"
-    document_id => "mdb_columns_cat"
-    doc_as_upsert => true
-    #user => "es_user"
-    password => "elastic"
- }
-}
-input {
-  jdbc {
-     jdbc_connection_string => "jdbc:postgresql://fda-metadata-db:5432/fda"
-     jdbc_user => "postgres"
-     jdbc_password => "postgres"
-     jdbc_driver_class => "org.postgresql.Driver"
-     jdbc_driver_library => "/usr/share/logstash/postgresql.jar"
-     statement => "SELECT * from mdb_columns_nom;"
-     schedule => "*/1 * * * *"
- }
-} output {
-  elasticsearch {
-    hosts => ["http://fda-search-service:9200"]
-    index => "columntypeindex"
-    document_id => "mdb_columns_nom"
-    doc_as_upsert => true
-    #user => "es_user"
-    password => "elastic"
- }
-}
-input {
-  jdbc {
-     jdbc_connection_string => "jdbc:postgresql://fda-metadata-db:5432/fda"
-     jdbc_user => "postgres"
-     jdbc_password => "postgres"
-     jdbc_driver_class => "org.postgresql.Driver"
-     jdbc_driver_library => "/usr/share/logstash/postgresql.jar"
-     statement => "SELECT * from mdb_columns_num;"
-     schedule => "*/1 * * * *"
- }
-} output {
-  elasticsearch {
-    hosts => ["http://fda-search-service:9200"]
-    index => "columntypeindex"
-    document_id => "mdb_columns_num"
-    doc_as_upsert => true
-    #user => "es_user"
-    password => "elastic"
- }
-}
\ No newline at end of file