diff --git a/.docs/system-services-upload.md b/.docs/system-services-upload.md new file mode 100644 index 0000000000000000000000000000000000000000..287d3a371780762f8213aec189a05d301e7c6788 --- /dev/null +++ b/.docs/system-services-upload.md @@ -0,0 +1,55 @@ +--- +author: Martin Weise +--- + +# Upload Service + +## tl;dr + +!!! debug "Debug Information" + + Image: [tusproject/tusd:v1.12`](https://hub.docker.com/r/tusproject/tusd) + + * Ports: 1080/tcp + * Prometheus: `http://<hostname>:1080/api/upload/metrics` + * API: `http://<hostname>:1080/api/upload` + +## Overview + +We use the [TUS](https://tus.io/) open protocol for resumable file uploads which based entirely on HTTP. + +### Examples + +Upload a CSV-file into the `dbrepo-upload` bucket with the console +via `http://<hostname>/admin/storage/browser/dbrepo-upload`. + + + +We recommend using a TUS-compatible client: + +* [tus-java-client](https://github.com/tus/tus-java-client) (Java) +* [tus-js-client](https://github.com/tus/tus-js-client) (JavaScript/Node.js) +* [tusd](https://github.com/tus/tusd) (Go) + +## Limitations + +* No support for authentication. + +!!! question "Do you miss functionality? Do these limitations affect you?" + + We strongly encourage you to help us implement it as we are welcoming contributors to open-source software and get + in [contact](../contact) with us, we happily answer requests for collaboration with attached CV and your programming + experience! + +## Security + +1. We strongly encourage to limit the clients allowed to upload by adding your subnet, e.g. `128.130.0.0/16` + (=TU Wien subnet) to the [Gateway Service](../system-services-gateway) configuration file like this: + + ```nginx title="dbrepo.conf" + location /api/upload { + allow 128.130.0.0/16; + deny all; + ... + } + ``` diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index a703eadeabf41b0f402e844e75dec406e93e9b6c..58e2f0cea52b241f3283596a73317ce193eb3091 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -179,7 +179,7 @@ services: environment: S3_STORAGE_ENDPOINT: "${STORAGE_ENDPOINT:-http://storage-service:9000}" S3_ACCESS_KEY_ID: "${STORAGE_USERNAME:-minioadmin}" - S3_SECRET_ACCESS_KEY: ${STORAGE_PASSWORD:-minioadmin} + S3_SECRET_ACCESS_KEY: "${STORAGE_PASSWORD:-minioadmin}" volumes: - "${SHARED_FILESYSTEM:-/tmp}:/tmp" healthcheck: @@ -361,6 +361,32 @@ services: logging: driver: json-file + dbrepo-upload-service: + restart: "no" + container_name: dbrepo-upload-service + hostname: upload-service + image: docker.io/tusproject/tusd:v1.12 + ports: + - "1080:1080" + command: + - "--base-path=/api/upload/files/" + - "-s3-endpoint=${STORAGE_ENDPOINT:-http://storage-service:9000}" + - "-s3-bucket=dbrepo-upload" + environment: + AWS_ACCESS_KEY_ID: "${STORAGE_USERNAME:-minioadmin}" + AWS_SECRET_ACCESS_KEY: "${STORAGE_PASSWORD:-minioadmin}" + AWS_REGION: "${STORAGE_REGION_NAME:-eu-west-1}" + depends_on: + dbrepo-storage-service: + condition: service_healthy + healthcheck: + test: wget -qO- localhost:1080/metrics | grep "tusd" || exit 1 + interval: 10s + timeout: 5s + retries: 12 + logging: + driver: json-file + dbrepo-mirror-service: restart: "no" container_name: dbrepo-mirror-service diff --git a/mkdocs.yml b/mkdocs.yml index 9549ef62010176bf27a797f447ed6c6d97c96304..a632fe06c95aba0b47f1c71965abd91e655e222f 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -22,6 +22,7 @@ nav: - Data Service: system-services-data.md - Metadata Service: system-services-metadata.md - Mirror Service: system-services-mirror.md + - Upload Service: system-services-upload.md - Storage Service: system-services-storage.md - Databases: - Auth Database: system-databases-auth.md