diff --git a/dbrepo-gateway-service/dbrepo.conf b/dbrepo-gateway-service/dbrepo.conf index a9d996358bc3ac4380627093d69bfa1e15f66776..944cab09389f49ea3991def5782d61cab7038c9e 100644 --- a/dbrepo-gateway-service/dbrepo.conf +++ b/dbrepo-gateway-service/dbrepo.conf @@ -34,6 +34,10 @@ upstream storage-service { server storage-service:9001; } +upstream upload { + server upload-service:1080; +} + server { listen 80 default_server; server_name _; @@ -82,6 +86,15 @@ server { proxy_read_timeout 90; } + location /api/upload { + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_pass http://upload; + proxy_read_timeout 90; + } + location /api/analyse { proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; diff --git a/dbrepo-ui/api/index.js b/dbrepo-ui/api/index.js index f6a62d6562c87fa77a619aa63fa88f52cb846a0c..9fb169e7b56434d32caef300446a106436677acf 100644 --- a/dbrepo-ui/api/index.js +++ b/dbrepo-ui/api/index.js @@ -1,6 +1,8 @@ import axios from 'axios' +import config from '../dbrepo.config.json' -const baseUrl = `${location.protocol}//${location.host}` +const protocol = config.api.useSsl ? 'https' : 'http' +const baseUrl = `${protocol}://${config.api.endpoint}:${config.api.port}` const instance = axios.create({ timeout: 10000, @@ -8,6 +10,4 @@ const instance = axios.create({ baseURL: baseUrl }) -console.debug('base url:', baseUrl) - export default instance diff --git a/dbrepo-ui/api/middleware.service.js b/dbrepo-ui/api/middleware.service.js index f732f25dfa391476e82df8ef0bd7a7bb5e8c169b..be56c223bb2fc12365eca24b246fd40262a5d80b 100644 --- a/dbrepo-ui/api/middleware.service.js +++ b/dbrepo-ui/api/middleware.service.js @@ -18,25 +18,6 @@ class MiddlewareService { }) }) } - - upload (file) { - return new Promise((resolve, reject) => { - const formData = new FormData() - formData.append('file', file, file.name) - axios.post('/server-middleware/upload', formData, { headers: { 'Content-Type': 'multipart/form-data' } }) - .then((response) => { - const metadata = response.data - console.debug('response metadata', metadata) - resolve(metadata) - }) - .catch((error) => { - const { code, message } = error - console.error('Failed to upload file', error) - Vue.$toast.error(`[${code}] Failed to upload file: ${message}`) - reject(error) - }) - }) - } } export default new MiddlewareService() diff --git a/dbrepo-ui/api/upload.service.js b/dbrepo-ui/api/upload.service.js new file mode 100644 index 0000000000000000000000000000000000000000..40362959068b85bad15f3f7b0df9e85b53240083 --- /dev/null +++ b/dbrepo-ui/api/upload.service.js @@ -0,0 +1,48 @@ +import Vue from 'vue' +import config from '../dbrepo.config' +const tus = require('tus-js-client') + +class UploadService { + upload (file) { + return new Promise((resolve, reject) => { + const protocol = config.api.useSsl ? 'https' : 'http' + const baseUrl = `${protocol}://${config.api.endpoint}:${config.api.port}` + const upload = new tus.Upload(file, { + endpoint: `${baseUrl}/api/upload/files`, + retryDelays: [0, 3000, 5000, 10000, 20000], + metadata: { + filename: file.name, + filetype: file.type + }, + onError (error) { + console.error('Failed because: ' + error) + reject(error) + }, + onProgress (bytesUploaded, bytesTotal) { + const percentage = ((bytesUploaded / bytesTotal) * 100).toFixed(2) + console.debug(bytesUploaded, bytesTotal, percentage + '%') + }, + onSuccess () { + console.info('Download %s from %s', upload.file.name, upload.url) + Vue.$toast.success('Successfully uploaded file') + const matches = upload.url.match(/files\/([a-z0-9]+)/gi) + if (matches.length !== 1) { + console.error('Failed to match file name', matches) + reject(new Error('Failed to match file name')) + } + upload.s3key = matches[0].replace('files/', '') + resolve(upload) + } + }) + upload.findPreviousUploads().then(function (previousUploads) { + /* Found previous uploads so we select the first one */ + if (previousUploads.length) { + upload.resumeFromPreviousUpload(previousUploads[0]) + } + upload.start() + }) + }) + } +} + +export default new UploadService() diff --git a/dbrepo-ui/components/dialogs/EditTuple.vue b/dbrepo-ui/components/dialogs/EditTuple.vue index 2a975d997a8a7df8669083cb1ec3eb422d1b3620..781b58014ae8d763b2df40a2ac0523fdc2eb70c8 100644 --- a/dbrepo-ui/components/dialogs/EditTuple.vue +++ b/dbrepo-ui/components/dialogs/EditTuple.vue @@ -132,7 +132,7 @@ <script> import QueryService from '@/api/query.service' -import MiddlewareService from '@/api/middleware.service' +import UploadService from '@/api/upload.service' export default { props: { @@ -311,11 +311,12 @@ export default { if (!file) { return } - MiddlewareService.upload(file) + UploadService.upload(file) .then((metadata) => { console.debug('uploaded file', metadata) + const { s3key } = metadata this.localDisplay[column.internal_name] = this.localTuple[column.internal_name] - this.localTuple[column.internal_name] = metadata.path + this.localTuple[column.internal_name] = s3key }) .catch((error) => { console.error(`Failed to set column value: ${column.internal_name}`, error) diff --git a/dbrepo-ui/dbrepo.config.json b/dbrepo-ui/dbrepo.config.json index 5683a2059536cf5fd21a4065f1f318cc9577d4a0..007198b35d83ec004997d9771529560fa7488592 100644 --- a/dbrepo-ui/dbrepo.config.json +++ b/dbrepo-ui/dbrepo.config.json @@ -10,6 +10,11 @@ "icon": { "path": "/favicon.ico" }, + "api": { + "endpoint": "localhost", + "port": 80, + "useSsl": false + }, "broker": { "connection": { "host": "localhost", @@ -20,9 +25,6 @@ } }, "storage": { - "endpoint": "storage-service", - "port": 9000, - "useSsl": false, "accessKey": { "id": "minioadmin", "secret": "minioadmin" diff --git a/dbrepo-ui/nuxt.config.js b/dbrepo-ui/nuxt.config.js index 9bab00a4e3e6396f6d29a6b68981af73ac6c712a..5b2c854f8716a510bf185b966a87b89bba5eb3fc 100644 --- a/dbrepo-ui/nuxt.config.js +++ b/dbrepo-ui/nuxt.config.js @@ -129,7 +129,7 @@ export default { accent: colors.amber.darken3, secondary: colors.blueGrey.base, info: colors.blue.lighten2, - code: colors.grey.lighten4, + code: colors.grey.base, warning: colors.orange.lighten2, error: colors.red.base /* is used by forms */, success: colors.green.base diff --git a/dbrepo-ui/package.json b/dbrepo-ui/package.json index 05054c6b70d7a1001775401205bd1baf249244e1..1b8975f1bd5475cfa1247b47d37def67472ca884 100644 --- a/dbrepo-ui/package.json +++ b/dbrepo-ui/package.json @@ -49,7 +49,7 @@ "nuxt-i18n": "^6.15.4", "qs": "^6.11.1", "sql-formatter": "^6.1.1", - "tus-js-client": "^3.1.0", + "tus-js-client": "^3.1.1", "vue": "^2.6.12", "vue-axios": "^3.5.2", "vue-chartjs": "^4.1.1", diff --git a/dbrepo-ui/pages/database/_database_id/info.vue b/dbrepo-ui/pages/database/_database_id/info.vue index de0a1082ed8366e3407daacf297b2465773f344d..567d7a71fcf8b8b8a5481d4ef006ddcef9f22910 100644 --- a/dbrepo-ui/pages/database/_database_id/info.vue +++ b/dbrepo-ui/pages/database/_database_id/info.vue @@ -78,9 +78,10 @@ </v-list-item-title> <v-list-item-content v-if="access && access.type"> <span> - <v-badge inline :content="databaseExtraInfo" color="primary"> - <pre v-text="accessDescription.text" /> + <v-badge v-if="databaseExtraInfo" inline :content="databaseExtraInfo" color="primary"> + <span v-text="accessDescription.text" /> </v-badge> + <span v-else v-text="accessDescription.text" /> </span> </v-list-item-content> <v-list-item-title v-if="access" class="mt-2"> @@ -308,7 +309,8 @@ export default { return this.database.owner.username === this.user.username }, jdbcString () { - return `jdbc://${this.database.container.ui_host}:${this.database.container.ui_port}/${this.database.internal_name}${this.database.container.ui_additional_flags} (username=${this.user.username}, password=yourpassword)` + const flags = this.database.container.ui_additional_flags ? this.database.container.ui_additional_flags : '' + return `jdbc://${this.database.container.ui_host}:${this.database.container.ui_port}/${this.database.internal_name}${flags} (username=${this.user.username}, password=yourpassword)` }, databaseExtraInfo () { return this.$config.databaseExtraInfo diff --git a/dbrepo-ui/pages/database/_database_id/table/_table_id/import.vue b/dbrepo-ui/pages/database/_database_id/table/_table_id/import.vue index fe1a4f503c915947b629dd623e81d07e478dc8be..c9221fafcbb1c8724d9bf75b660af346fe5ff33d 100644 --- a/dbrepo-ui/pages/database/_database_id/table/_table_id/import.vue +++ b/dbrepo-ui/pages/database/_database_id/table/_table_id/import.vue @@ -104,7 +104,7 @@ <script> import TableService from '@/api/table.service' import QueryService from '@/api/query.service' -import MiddlewareService from '@/api/middleware.service' +import UploadService from '@/api/upload.service' const { isNonNegativeInteger } = require('@/utils') export default { @@ -187,10 +187,11 @@ export default { isNonNegativeInteger, uploadAndImport () { this.loading = true - MiddlewareService.upload(this.fileModel) + UploadService.upload(this.fileModel) .then((metadata) => { console.debug('uploaded file', metadata) - this.tableImport.location = metadata.originalname + const { s3key } = metadata + this.tableImport.location = s3key QueryService.importCsv(this.$route.params.database_id, this.$route.params.table_id, this.tableImport) .then((metadata) => { console.debug('successfully imported data', metadata) diff --git a/dbrepo-ui/pages/database/_database_id/table/_table_id/info.vue b/dbrepo-ui/pages/database/_database_id/table/_table_id/info.vue index e5a5e86ff2b4c8c426632e6ebb91d88bc08ca69e..4f71b2c489e5545f49d03a82ca3c5776cdd27f41 100644 --- a/dbrepo-ui/pages/database/_database_id/table/_table_id/info.vue +++ b/dbrepo-ui/pages/database/_database_id/table/_table_id/info.vue @@ -36,9 +36,10 @@ </v-list-item-title> <v-list-item-content v-if="access && access.type"> <span> - <v-badge inline :content="brokerExtraInfo" color="primary"> + <v-badge v-if="brokerExtraInfo" inline :content="brokerExtraInfo" color="primary"> <span v-text="accessDescription.text" /> </v-badge> + <span v-else v-text="accessDescription.text" /> </span> </v-list-item-content> </v-list-item-content> @@ -59,7 +60,7 @@ </v-list-item-title> <v-list-item-content v-if="database"> <span> - <v-badge inline :content="database.exchange_type" color="secondary">{{ database.exchange_name }}</v-badge> + <v-badge inline :content="database.exchange_type" color="code">{{ database.exchange_name }}</v-badge> </span> </v-list-item-content> <v-list-item-title class="mt-2"> @@ -67,7 +68,7 @@ </v-list-item-title> <v-list-item-content v-if="table"> <span> - <v-badge inline :content="table.queue_type" color="secondary">{{ table.queue_name }}</v-badge> + <v-badge inline :content="table.queue_type" color="code">{{ table.queue_name }}</v-badge> </span> </v-list-item-content> <v-list-item-title v-if="table && table.routing_key" class="mt-2"> diff --git a/dbrepo-ui/pages/database/_database_id/table/import.vue b/dbrepo-ui/pages/database/_database_id/table/import.vue index 8c8d566d8eba6095ee732d9f6ba20b66a6d72761..dfd826ba9ecaa0a1076a94261c2c90d6bba75c81 100644 --- a/dbrepo-ui/pages/database/_database_id/table/import.vue +++ b/dbrepo-ui/pages/database/_database_id/table/import.vue @@ -163,7 +163,7 @@ <v-btn class="mb-1" :disabled="!fileModel" - :loading="loadingUpload || loadingAnalyse" + :loading="loading" color="primary" type="submit" @click="uploadAndAnalyse"> @@ -177,7 +177,7 @@ Table Schema </v-stepper-step> <v-stepper-content step="4"> - <TableSchema :back="true" :error="error" :loading="loadingImage" :columns="tableCreate.columns" @close="schemaClose" /> + <TableSchema :back="true" :error="error" :loading="loading" :columns="tableCreate.columns" @close="schemaClose" /> </v-stepper-content> <v-stepper-step :complete="step > 5" @@ -203,7 +203,7 @@ import AnalyseService from '@/api/analyse.service' import DatabaseService from '@/api/database.service' import QueryMapper from '@/api/query.mapper' import TableMapper from '@/api/table.mapper' -import MiddlewareService from '@/api/middleware.service' +import UploadService from '@/api/upload.service' export default { name: 'TableFromCSV', @@ -261,9 +261,6 @@ export default { skip_lines: 1 }, loading: false, - loadingUpload: false, - loadingAnalyse: false, - loadingImage: false, warnAnalyseSeparator: false, suggestedAnalyseSeparator: null, url: null, @@ -320,30 +317,40 @@ export default { isNonNegativeInteger, uploadAndAnalyse () { return this.upload() - .then(metadata => this.analyse(metadata.originalname)) + .then((metadata) => { + const { s3key } = metadata + this.analyse(s3key) + }) + .catch(() => { + this.loading = false + }) + .finally(() => { + this.loading = false + }) }, submit () { this.$refs.form.validate() }, upload () { - this.loadingUpload = true + this.loading = true return new Promise((resolve, reject) => { - MiddlewareService.upload(this.fileModel) + UploadService.upload(this.fileModel) .then((metadata) => { console.debug('uploaded file', metadata) resolve(metadata) }) .catch((error) => { - this.loadingUpload = false + this.loading = false + this.$toast.error(`Failed to upload file: ${error}`) reject(error) }) .finally(() => { - this.loadingUpload = false + this.loading = false }) }) }, analyse (filename) { - this.loadingAnalyse = true + this.loading = true AnalyseService.determineDataTypes(filename, this.tableImport.separator) .then((analysis) => { const { columns, separator } = analysis @@ -370,7 +377,7 @@ export default { } }) .finally(() => { - this.loadingAnalyse = false + this.loading = false }) }, listTables () { @@ -393,7 +400,7 @@ export default { this.createTable() }, async loadDateFormats () { - this.loadingImage = true + this.loading = true try { const database = await DatabaseService.findOne(this.$route.params.database_id) this.dateFormats = database.container.image.date_formats diff --git a/dbrepo-ui/plugins/axios.js b/dbrepo-ui/plugins/axios.js index 8628b6214e6eeaf5b0a4ab0d9285e3918fc76ff9..0f67762dbfef1f7e33878dd0eaf91978056b8e60 100644 --- a/dbrepo-ui/plugins/axios.js +++ b/dbrepo-ui/plugins/axios.js @@ -23,7 +23,7 @@ api.interceptors.request.use((config) => { } AuthenticationService.authenticateToken(refreshToken) .then((authentication) => { - console.debug('interceptor inject authorization header for url', config.url) + // console.debug('interceptor inject authorization header for url', config.url) config.headers.Authorization = `Bearer ${authentication.access_token}` return config }) @@ -31,7 +31,7 @@ api.interceptors.request.use((config) => { return config }) } - console.debug('interceptor inject authorization header for url', config.url) + // console.debug('interceptor inject authorization header for url', config.url) config.headers.Authorization = `Bearer ${token}` return config }) diff --git a/dbrepo-ui/yarn.lock b/dbrepo-ui/yarn.lock index 545ee88320ff8d81b43be9feb331eb766d215534..d5a3920df7018fd57ec7b47837ff13f92cd42e7f 100644 --- a/dbrepo-ui/yarn.lock +++ b/dbrepo-ui/yarn.lock @@ -6867,11 +6867,6 @@ fs.realpath@^1.0.0: resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" integrity sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw== -fs@^0.0.1-security: - version "0.0.1-security" - resolved "https://registry.yarnpkg.com/fs/-/fs-0.0.1-security.tgz#8a7bd37186b6dddf3813f23858b57ecaaf5e41d4" - integrity sha512-3XY9e1pP0CVEUCdj5BmfIZxRBTSDycnbqhIOGec9QYtmVH2fbLpj86CFWkrNOkt/Fvty4KZG5lTglL9j/gJ87w== - fsevents@^1.2.7: version "1.2.13" resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-1.2.13.tgz#f325cb0455592428bcf11b383370ef70e3bfcc38" @@ -13042,7 +13037,7 @@ tty-browserify@0.0.0: resolved "https://registry.yarnpkg.com/tty-browserify/-/tty-browserify-0.0.0.tgz#a157ba402da24e9bf957f9aa69d524eed42901a6" integrity sha512-JVa5ijo+j/sOoHGjw0sxw734b1LhBkQ3bvUGNdxnVXDCX81Yx7TFgnZygxrIIWn23hbfTaMYLwRmAxFyDuFmIw== -tus-js-client@^3.1.0: +tus-js-client@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/tus-js-client/-/tus-js-client-3.1.1.tgz#87cb72e528d274d0a8ff62e9c18165f1e901ce9e" integrity sha512-SZzWP62jEFLmROSRZx+uoGLKqsYWMGK/m+PiNehPVWbCm7/S9zRIMaDxiaOcKdMnFno4luaqP5E+Y1iXXPjP0A== diff --git a/docker-compose.yml b/docker-compose.yml index f38cb5618f150c5e5e1c439efef7e20aff3c8045..c0074047d7bf7695d3d1546b857b36d70d1a170a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -183,7 +183,7 @@ services: environment: S3_STORAGE_ENDPOINT: "${STORAGE_ENDPOINT:-http://storage-service:9000}" S3_ACCESS_KEY_ID: "${STORAGE_USERNAME:-minioadmin}" - S3_SECRET_ACCESS_KEY: ${STORAGE_PASSWORD:-minioadmin} + S3_SECRET_ACCESS_KEY: "${STORAGE_PASSWORD:-minioadmin}" volumes: - "${SHARED_FILESYSTEM:-/tmp}:/tmp" healthcheck: @@ -368,6 +368,32 @@ services: logging: driver: json-file + dbrepo-upload-service: + restart: "no" + container_name: dbrepo-upload-service + hostname: upload-service + image: docker.io/tusproject/tusd:v1.12 + ports: + - "1080:1080" + command: + - "--base-path=/api/upload/files/" + - "-s3-endpoint=${STORAGE_ENDPOINT:-http://storage-service:9000}" + - "-s3-bucket=dbrepo-upload" + environment: + AWS_ACCESS_KEY_ID: "${STORAGE_USERNAME:-minioadmin}" + AWS_SECRET_ACCESS_KEY: "${STORAGE_PASSWORD:-minioadmin}" + AWS_REGION: "${STORAGE_REGION_NAME:-eu-west-1}" + depends_on: + dbrepo-storage-service: + condition: service_healthy + healthcheck: + test: wget -qO- localhost:1080/metrics | grep "tusd" || exit 1 + interval: 10s + timeout: 5s + retries: 12 + logging: + driver: json-file + dbrepo-mirror-service: restart: "no" container_name: dbrepo-mirror-service