From 35fffea7405668781022c9e7a80a34018dc03e7f Mon Sep 17 00:00:00 2001 From: Martin Weise <martin.weise@tuwien.ac.at> Date: Fri, 9 Aug 2024 09:24:19 +0000 Subject: [PATCH] Master --- .docker/dist.tar.gz | Bin 9967 -> 9975 bytes .docker/docker-compose.yml | 1 + .docs/.swagger/api.yaml | 6 +- .docs/api/analyse-service.md | 7 +- .docs/api/gateway-service.md | 6 ++ .docs/api/metadata-service.md | 18 +++- .docs/examples/health.md | 25 ++++++ dbrepo-analyse-service/app.py | 1 + dbrepo-analyse-service/determine_dt.py | 10 ++- .../at/tuwien/endpoints/SubsetEndpoint.java | 38 +++------ .../at/tuwien/endpoints/TableEndpoint.java | 10 ++- .../at/tuwien/endpoints/ViewEndpoint.java | 16 ++-- .../endpoint/SubsetEndpointUnitTest.java | 45 ++++------ .../endpoint/TableEndpointUnitTest.java | 11 +-- .../tuwien/endpoint/ViewEndpointUnitTest.java | 5 -- .../tuwien/mvc/PrometheusEndpointMvcTest.java | 8 +- .../service/SchemaServiceIntegrationTest.java | 31 ++++++- .../service/TableServiceIntegrationTest.java | 53 ++++++++---- .../src/test/resources/init/weather.sql | 9 +- .../gateway/impl/KeycloakGatewayImpl.java | 6 +- .../impl/MetadataServiceGatewayImpl.java | 13 ++- .../java/at/tuwien/mapper/DataMapper.java | 33 ++++---- .../java/at/tuwien/mapper/MariaDbMapper.java | 78 +++++++++++++++++ .../java/at/tuwien/service/ViewService.java | 4 +- .../impl/AccessServiceMariaDbImpl.java | 25 ++++-- .../impl/DatabaseServiceMariaDbImpl.java | 4 + .../service/impl/HibernateConnector.java | 3 +- .../impl/QueueServiceRabbitMqImpl.java | 2 + .../impl/SchemaServiceMariaDbImpl.java | 12 +++ .../impl/SubsetServiceMariaDbImpl.java | 48 +++++++++-- .../service/impl/TableServiceMariaDbImpl.java | 29 ++++++- .../service/impl/ViewServiceMariaDbImpl.java | 17 +++- dbrepo-gateway-service/dbrepo.conf | 2 +- .../impl/BrokerServiceGatewayImpl.java | 8 +- .../gateway/impl/CrossrefGatewayImpl.java | 2 +- .../components/database/DatabaseCard.vue | 22 ++++- .../components/database/DatabaseToolbar.vue | 49 ++++++----- dbrepo-ui/components/identifier/Citation.vue | 27 +++--- dbrepo-ui/components/subset/SubsetList.vue | 16 ++-- dbrepo-ui/components/subset/SubsetToolbar.vue | 16 ++-- dbrepo-ui/components/view/ViewToolbar.vue | 13 ++- dbrepo-ui/composables/axios-instance.ts | 2 +- dbrepo-ui/composables/identifier-service.ts | 2 +- dbrepo-ui/composables/query-service.ts | 10 ++- dbrepo-ui/composables/table-service.ts | 6 +- dbrepo-ui/composables/view-service.ts | 4 +- dbrepo-ui/locales/en-US.json | 4 +- .../pages/database/[database_id]/settings.vue | 2 +- .../[database_id]/subset/[subset_id]/data.vue | 9 ++ .../[database_id]/subset/[subset_id]/info.vue | 7 +- .../[database_id]/table/[table_id]/data.vue | 40 ++++----- .../[database_id]/view/[view_id]/data.vue | 2 +- dbrepo-ui/utils/index.ts | 18 ++-- helm/dbrepo/templates/auth-configmap.yaml | 4 +- helm/dbrepo/templates/metadata-secret.yaml | 2 +- helm/dbrepo/templates/search-db-secret.yaml | 79 ------------------ helm/dbrepo/templates/search-secret.yaml | 2 +- helm/dbrepo/values.yaml | 19 ++--- install.sh | 24 +++--- mkdocs.yml | 1 + 60 files changed, 586 insertions(+), 380 deletions(-) create mode 100644 .docs/examples/health.md delete mode 100644 helm/dbrepo/templates/search-db-secret.yaml diff --git a/.docker/dist.tar.gz b/.docker/dist.tar.gz index 411c92f64cd600841b98f1375c32b920895df2e0..13ab9159e781ba44af31da5ca522d76e0b5edcd8 100644 GIT binary patch delta 9976 zcmaFw``x!*zMF#q44B{8WEamqvFZ2C{Y{_dX{k#uUjFvRuBWTIRn`c+PF1;CxXo`P zhntWYljMY)_cuS^_vhC?XrQ2S`{XjSB}s<;?`FPy_wwbtnF|9#e4oliu1Pv4zO0w) zeDUAu;dNdg=E%rCy#ML=<Df6%^A;=r`0xGm#giX9s$wkb5B_|**lX(_?%=AoeHZ2} zeS1Re@E@akms!1+Ugv84cbq?UrT+61>)slkk+IyQWo@*&FY-_K=M!7@l%!q!a>Hoj zhZ`rNGP$@mPv^cn<*LiJ-W2c7gSY!KPX<p}v2x{!0+Z)6HEw*KoL7H!W2=~&^h>jI zsS@0Y)2vL>?i+l+D|6JOUMA9bf~w>vnX4KT^0v;*w2oul<TBe?o$=gUE!ML-eWo$N z+UHWeCog<hv@<|2BvQ23^k%~8)N59SjLJ`r^3PT=>eN~jso}I;{nnO<X*o}H#Mi7n zb0J3W27i_Aj4ci-uPYO?+74&6JzMd($!q02m&lK=AIRmnc!ZT*)b%;t$y1-bKT3D* zl@Id$k4{8nPRl&Q7$30ma3|CCHQv5?C$8>f<p1;YdcD=335O+vHpeB~Zu?U{pFyB{ z@4_q3*_vn1yP>l18f)PP$w{T@Y$^fon)xgg#1Gs`muOmZ&a^o)<JgqMo2ndFcc#wb z+oPXyyUs-SYfZ9tz|CtV67&0IJ=ivB*1tAg7PF~YY^$Zw^b0QvCnPH#zvLh{qv+__ zi)UYoR0);@ZrvifDeKJKT&XuK7v8UZT4OG`{Pg$RC(7)0vYq0yD^$8aWnJ~#!1nbU zUDNmd$mjik*MIWb{+}t@np*k#YW2#3AGdy<ru4{JP)X=)t*v0^TKN#cg$Dmi*pD9S zZJTdUAHGXw+J@>cYYudk%y-_-(vYui@KQI0<IKm~oIU%Oy;Cx^)9@`><ht)qM8>L{ z8}^A9d@rqxDVWwTWyG~9Xu{k@?b{_@2kk5fu{gJE_ogi+ZClxMpCvrA=#JVLp|#_G z*7Bv7gDx`d*!fo3KH*=|vJZ#eRKKi!wtm0FmnD`yKV3!YmvbyV|73+n;E@lT8d(!f z9=L}4N$TisT$Jk~obcne+wSABUNM`MORgI<{Ey1`5ch4D<u%dkA{kzn3r$|fxm;Y4 z(a3Xknve1nd;d6(Zas}MR-dafyb2QgeLn5@D|1}na-Jo__8II8L$}?IQ8;*c&B@s3 zE{m#PS5CaJVfOU!SzGFN$vm~5bK%<!S<lUFoO3VC{~vEI8hLy%mulPVqPMph<+<4I z=YFepDu40NLp$YfrIz%je>v^j_FKnaUm?sCo}7^+;SsMD;nG*!X~-M6SocFk2n&<_ z?fI>17BN4b!^1Kmr=vz{-hwG_d*oc7KK}gV^n=fjrw99qTAP0gU3|;6yZ+PHyN|cH z<|g`9F;z}_6Uh4d(w*b&*Zne8s)Pj8m^P?<f1UJxlDO(W@5jQ|b3fh-;yxSvnK@Ln zeXrEV#ViRblli``H2>hDlc6b-bVpXTT52kbk@mU23%029EuLC9$=2>?gyO4<GbV(I zTrj?OHEL#4*<QVt7an(I*>{K?dDxL%Klw{%+GKAx=Qq4;O7(*Ccb~ko;PyJ<6%osF zqcegG)@F8`+}fNuWr4VhV9?QxC%6ApTwypdzvWnsO?RxqX|}hW>9H5Or=2Ku&)x8@ zZM&X_$=Q2{4wqTwY_^<pHtTcc*=_sVCd|)2AS81&QM80n*>jsRN9n`9*VmSR=igkt zw)9(l#H!-(Kdi^UO5M4t*?wC3vUkPvo$c(8FNZFzD=seHmQ{8;$+<GC?#G8OqJMq} ze_og@Hg(fK$8fzLyRH3x)O@cD{PQK@Yrgx@i;>x1H{^U&|NHT4tMQ-D%W77}oa)~@ zn_qnXZrgRYGG;ChfBKNs{^#cvQTAK+>FK@g{`ma#)zH`VRiXc?zIg6?v;NlNpYCgo z?f-7E($>x`-D#3P&u`tTvv#Z2EPI;u`qQNUU3PtX%QkoOFMW9P)0F*xCTrJK?$G*b zve0n*^X-2>KF%t-wzf}tdfDEJXuXfyKd*oEBV>l@nb(_t?kTsanRU-+YpS?uo8R`0 zs}}rw>%7}S>+YI)(c<Q+^&58NJp3f$yL!5q`7Y~Ry?ND_>)!9m3;9}={b^xUrQwdN zc2kb^#~+`+|DH{c^ilm^wM(xUo!&0|Wv4#ZIrqeWSJu>sbXR@PyY)uYy5DK??nkc_ z*WcPZ?aH(DKh37CFKxQo<GDg-(Zbf=g>17{&o{m#(8#AAXQM8a;l)&&cTIXhy-vH5 zdSOzwK)X8kO_e>%mn?FaH}8;L@FugX*F=8S<ty-1KU%xKn^Ah^akE5*<eP3i9Ld{F zk8hdh{3Xf1M|(m<oAuS)JzsRTG^IuvZZ&*hxzRED_y)zvL1HiX?B{uyDzJriW}Eyt z=+LC8!zpJiJd;7&UTKNnn;p75qOTVpQ?0k>Iq*m1WR*te_Z?m@Q*Dk#T5C^Dy8GqY zUytKVJ|~;5y!&SPah0xV)m^>&5?p)NnTS}`<Uc9b+B!vddeYpTvOGWiT-*!{Cc0bj z))nSlnowytL8dPzBdbDh&Ke7E$&83E4&7EaKC*3kdtj!u#8>`JGCOsXbXVT<z7%-O zKkj^MeZ=eSZT`E|Zab(mJ+W-q-l-^%_GemRi-n1#=YlDDqCN`0wtbs@Ddt;#)~9;= z=>N+%ets-r{eR8WH_|$el7^GH7oM%&aB|1VZ`#w@KeD~PBKB(Lwr@|aohwXmvJB=5 zHs{i*<XQi@J!)&fx|ZK8qEScWtq+<0X+O8RfZy05^1#dbC_ziF8KU~$O{ZRYnTaeB z+hfA$Gezy-M8*EnC0^XMoG}|X3l^Dhu3N-;FS>Q>b-jeXs|WPrREnG}Zi!VltvhNN z;WF!wo8pa-ll-&NLw)B-DM-#rGJeCn_~xXoCOmuig*f)78Bboi&fYlrgY^V1@7<?< zl=AJn8pCm3KF5;Rsoo-q?a1fthCbJ9Z)SBTuZ=&awKT|Z!p21@$}3pb#IByRn|&F> z;o@lv=i0p5-*HZ)yHnm{{?2T_S6|z2<fraG^LDW&v%SCjWe4lzZw-1EE^;|G|2Cg< zgSVWk<nvCkZE3T;y(7<;G+cQ$OW5<VuvYO0tL!s@-ODV#8T!T?TC$?D{?QN5!pvyb zYPIMU9VIuX#%^zto^&g4OIh&d!h~HD-exfg%l@7ndEmb7i@Ptj-kulSGtnjJ&3pD$ z@*gcMYzj8`wa2rbp2Da<Pj7C<iKZh$BHN=mP9J01|8Q;mkI2jyA0OQBY!F%d`(^7v z0ioFr7Zbnys=3}jt-^6d{onR8QQ3y|Tmb?v)iy`i%+>h1cwwf!_O6MN{eJEf-JCvT z@0g-dQ#K{PU-8_I4Q2+e6W8ALJHc|{$eYDe^NjzV*q3^4zx?q_$IFa1<o?h+u;JW? zH_xtr;rbW&?N`w4pt|W&iks?=nFVq6rg5c}e>uu4u%)%_$k*0_+!NaaJ)Dw7k9Y5O z=>Jv!R_^+ln>Kvr;(I5raP>*4KFrB_OX%I@xh8$F*A%8Us6RW?z4rBS&7uRd-fa}L zTzklc$A4ph)2Es4QTN=EmTzj7d-_~9SDtCsgw&|aLnloZEcaJ9Ftxu`Nc%N+2Fsl@ z#@E=mCuM2Odvw_G_v<eg1GlpM58iV2R$+0q%t!OPZFlQ0HFe2aRIaqno^EVA!>4=3 zGl41lJxylSMWjjdw8(Gc%~&8;d>}{hP|10V?L3nov<L)u*J|>~R0^lKhCg*(8)Tt5 zXPvUjnoz&sQ*YI-tM`{APSr4RwB$N`d(QEFN*PItTSQF0OqkWrw%zcA@aE!d_7!Zi zoy+W8yx;Wxnkej5A2)03{9N|4k3&D6^!sITM<^`xob5Z=L$lvpIC1e!njn)`<WY<A z>PnB}(*jh#Wb?WPacr92S{w7UFpY`ja9vW&%FZ>_TZ_Ja=}DY@CeJqg#_<x5NtUmV zswv$6w%T_V|Jyi+iiyf^lTLhDcHOkBeC6J{v|0Z4w)@xG?DrMBb^A?ya?siBe<q2y zt1aydnq`nYW#-bGzb?%aan@fErYkf5=h<`5H!Zw2<NnOr9@fa;?Pu&(XI<O*I(e7o z?pOasHXJSPo4ZZnuyNwKy8)YHr|wxIol%y3=3`69n|<tMVfWJi?L8>brMxmwpxxk# zcid}@Xr-R;Ly3EW1de%MZT6Ts^US__CK2YT!3Ek4Ur#5TGW^E#vMNJ;cdCVi?ex35 z_Rsu&|Dd1R$qn6gix->_+bU}+vv&1Nt#@tDH^!SanZ`_EGY@J%oPY0--J8ys>{@{v z+FdJlx@=e&wKh-ksz<n_wN>iviZ>fi_uZ~~Z<ugtvcH(xmbT4X#HUrZi?HuswKbIW zMo4|v%bu@Sl8$FAlhooC@INoe-ZA&I|C+UbJI`q^eb6bEtUfvR#JM*%4@1Nke|@9H zom23hf8Kh@{jnt{j~mAw;*h;>=4rp9&bY$!1W)t+L(5Fvlt1a3cgP4<=xbY?e6zDX zUv+&ntKgl4>FYl^t9CAZZk=&dJZATa9Yr4(y%x)_uD{w;w6U@3U|UIp(R#C-VpsLl z`Ry)__g|cJ5PbJ#N#d;?$DLi}_J?eGs^|YLw)HRXbWx5MFLGP|u6fPHa_97W?VPgy z_?qOz`p;8Yl_bJTsvpFsC|GYg%Q#Q`PTT^Sile(LPByPof9Du3ucOI!^UwJY_rH~w zKUfql))q8*{{y%Bw~9JaF~)m#I;3o7mdp2F=5Vj$(1hg;RX^?CC^8pMk^Xd0zgYK! zd7=|b6zAPi#R6WJ4&nTnisAX6o1K^4&5mi~TzLD*ei<X@z>KXGg3B9OE*x2Lzhw4@ zciDgTBz|O;vpQ3Hm}#2CjO^Zt8cARI*E2`37BUE^xkgDw^Bvr&U;pw!X6l<l2D$y# z)9NyPp6oT6_@C#~`Q3~iE^^irOQl;C-kD^y9!oHJo2>um7xTRHchath9Wbu<_@X=S z<guC=@*fml^jC^Myn6mq{rh}@zcu^r2WyBO@T|OTwrT(WP_5~Hr`~<vujT$U>*<nD z-Ya9)$%WU~{eK;8v)?E<AYs~?`uD8rn&(b7i&Rfs&p7AyY1Y2$>+IP!eAgDPxzjRp zhS=jX0Xxo$u24SqTaU|5bo=(i^pm%zE>b*tPL?;Z?fz_swB0xE<ZnudI5ua;ist6J zZ*p(<EthY;<$YS+{<Po|^>WKY|9)_n8qBf1b*e%xUV`ydtUFI+f6qUGX}#@B(-ddd zPn1kmlWw>oFQQ}NZQjVK{xJ4XQ<lfB%zOIcmm-X>IC&&FEX&&FEHhhun&1A<_oEZ) z?biQl`E8khgDvcne1HGtFZOjDf8#YDzPFd#pE&=}bNO}uo&Sk1n^oL@;Sukx8yo%w z|A@D3dNlv_fAb2XeMUdCa#KG4|FI%Oll^}ETDkhrg>jOf>gxV~oo!RPPqpNKe!f1( zkB2AZlw%C;-kb1nKS%S!hM&pn`c!jrZq&co``IiwMD1V0)5OMX9<i%l+vXj-Srcws zd47k-uEk})FSH)g`0&n$E#&EsAGVC97j^Q3)um6Z?pgowNyeeoJW1<b-dw!k%keE! zetg(GYl8ZNgzS2wZ)eTJzwK|*XibY;_pf24xa9k1fd>EFR6f6RxW3Hzxq`}E!`P(R zCe6za1WC!Iuz37hbm8E|B!l>w<+ENr@7s0gYDfA#Rwjp(-8B{azt7t#*f#%<0$=gj zZsvv8MR%>0IrK8`-BOR|5wn}_o|)ip!L#9B^`r}}YE}<s$o#UNaIao3(XQ!NVc9c< zJ5%gGTsWd)C~qGAquZ@<!xQF@KNS+UpXoOBWmHJrVb`Jd+M{q=*3n)mPp&ft*6Jtb zT7@mKEJ|B8YrE)CC!>8buWT*c*ltzs^GJ+!_>uSiS*#1AM{)2`^UYUGn`|z1?ktrs zcp!QtkZHZp(#$ilOM;Xq<Q#Zd|M_fru4+Ne^G%ccdA*&hxfqJPIjWePUPotrpPA!5 z!~5Y3pJ4Y*2L<>;y;Sy>?>@m9E23AWq`zfXonDHuQvJd18u6N<w6l?29|ThDdM14^ zP&nJ&qGiK8aUa)vt?;@C$F3KvW6pHH-k!T7?^dVXGo~*kF@J6@_;{$4tDgU~czwRF z*B6ENt2!B-bvoZYdCSelv+=|#@xAPPJ|&e|#(Y(uy#GcwZ+w?FQ^4_W#_>OPvg`T} zdPJ|CU~=}2t^T$1EK>82Z@83URT4SXsoQXV!5YTx#tT1bFwf76lzOPTS^9N<LLB#5 zm0~0Hg7`PFpJVxcwXBrnI?pQ5Y%Zd(%}}*|Q~P=DDM~(1gHEa+D7_*Ud6@qKQ)C6d z&OBM=MHgxWO-p<o+hn)CXgYn(^6QaHm#zhI#rklRC(U%(mXyhTJ>Wd6!wH`Y_Lm-Q z-BK>9@S;2QaIfETjeoHv`!+RmDO{=5a}&t+_H#bOos!Qi{50N2M{wWb>6uel9<951 zDan%UV*O{&Nx7XZr*bv=estu;Uccg-Y<2By;hi)|`{iGlWzy~+tTDOv)hK(HS|igm zd!G=A^02czzSZq~bS^dWxXB8qz<JrTkF_Rs?SDA8oLg=3zcV&U$4nkvzhBtIk|aC1 zRsPv_rL~O{4t18@dEgp#oT)0GU37UZ+nd#AYEH)WeH46OAHcdIxt{5I>$D59c{6r0 ze&IW<nD@fK<-?+jJMLfDuyVD`<Yi4pQ)gUWbNSjSpCF67JH&P$t1g-I<K2@NwOTxx z3eOuWBNnN4MO<O*@qRBNaXQB8fbdH>TlIt{!Q)?JA{LjFyb3p*R(#S({_O%ze}`-5 z7A$G@DoN|ywDXUO>*>Yysh_gC3fmT3++?4#WkPKA1Ag)4x95NQ-N3a>cxl-hftMWe zJ=1!Yu5)KEmdQx@9T(l8o1kEAIfdcI#%Uc~G9d;0Wf!%MoRO|*`!iiJG<w;J(>l2? z-36Uwz8pPbAbU)<=7HFQg8|dKr!Z&>&g{{b*_q%`ZR5SZIZ%9i-kqQ267}=tOk!VN z^ImZxVCT|RH?{}A&@VpnQN1DPYlM$-`f7clDQqwO?{Eg1TZxJ|Ep`1G+|@3BHR0p5 z59+FO4p<-Hzb{)lqnv%muY(U8`8o1S7jjQ}nA-Z@d0};Q{`Xh^wC4r=xcd7ew^dEB zMCl(J{fE=HRNtL8zd!zO%pJxbk{|T=>Jwi)ZhQE+Wc~GHt+MOxek-!+xwkw1_)^h5 zF<Yu-{^fpiI}rWnkh#M?L#CRzoBt2GAG>R2^(Wy(>92p*f9`xT_;f%nDm#$tv(j$X z2eph3@18GC$Yn15A*LTxu|`s>$}1~fNICIk&~=8I$=abYLV=5CEQ_~V_GB){ClduV zgY%{Jj|Cc!6_;jb*d~Xno!pgYmvllS!6|f|=v}UkX8z4B{^IjLi%K0?aX0JJ<z-pL zK_{1m25*gUtunnGs4BWv>FTuNl%IS~ZW$`O;&McJ6u0EZbOwd2NDtgOYv#++zmLE9 zG)tT8Q2YNYZ|C}-DlBGfM<wr`<Sz|;)DWU4e#QE7OTE0#r<i08W6me?Qyja~P6~<b zzAti=xA6GX`F-m*H$C9}K0l?#Qj)h=F5u$AAIx^GLEa8)U8byCKfm~m+S0e7sb?#S z+~gnCEfv1BR&Z9V;jPvK>m40k%i^3I-xO&aO!Jx3r1QGzRG&)pv^O)CRdoLUd;h-( zm-M#elMY(a@7WaVbF&h@>n-^9#BRoSLwmd1IXt1PqQ}Zr=lUE;anOG@S17JkQCGY| z+xV5E;YThli|Ci4PPf}TUA2Ok_-3et#08o7ZTR<akw~lX;lx~rPf>H?B5ujVtkYO7 zKI5{TzF9S^%TbRr)1riwA1L)UNV2)aF<ok|_!qNCsj~T`a%4tRy-te8$Gcx`j?4|% zv7vLvyUO`zQ~Yx{A8+F`ym$Ej*77yJxymQs{}a^`?wYV!^|XV~f*%eII#)iPV7z%Z zMr-LLUCEM(s_y(f<^}uz?s~4+yJVtHv*O9M&$%Nm-!;B$eEIW>;{rmP^c0h0i!`;e zP21V{g1?_~FPWT^<#TvuxcjmCd7pLts`Tc?{!<NF?UZqKV%?*OIjcgto~uleTFsvn z{_m2bQN+?cU8~!E^CVkceo=91!O~E@r#Jg2D;~M1zyI;P%7<$ug0?iQYK&~&%P$f9 z^v%k{@;#BdS9R0Q30-}>#^?U2^%mVyM|dU&34E+#Ht*70%HncIENxv{y7`ZptLN2= zEpE{N@uR{{ZTX~03ukUJHvfHtajD7tt=sQ8DqRoQG|BsD<kgQ!e$S0xp7<4*<KgFh z{umq2V*BpG{HG$jJwujX<}~Oz^Y_;quESOm|GVBQ1l~ze&*3~~e#Bti*$+``LZ?=J zY)-fvP_%LOVT(iTnLgjkAJqHySh4Nl-**43*3tS`E4F9pY)`SVnOOA7uz6p4mBa@w z34wDtcl46CmAETwRqa^z{K20;Z)%n<UGpGVHz54a_ZMIO-t0OeH7V_bcWF?tn~z<~ zY4*O?74et%NZpY1QI6DkD(u*oY5DMksk+Im>Y0H)s>|M{PFZGZ@Vt`oRrSR8zry`i zrp~ai(_%lwQQvo^Z$a$Cn40L+h<48xE0`7-8Yxd{=2W=6s^`pu+1Vc~8YP8ZF1#(J z@N4UuIlE51<}$6D%JHaL^3>!_yyyN@3x`|YwA7!Rr9GX~RPb-(si#f7-;<}D_9>sx zHMg}Wa#hSc$Lv4cE301X94omZHZA1OzZaa-qF#pW2$*u?^2y+*^*m?Z7|Z+3e*4W! zg>n9y(4B#)dK)Ww*Rw8fed(Q%Y0B-_qPa12;SRfx9VTjKRrk7*tB*Z$Npg<hUBCI1 zy5Vi5Q_FqItmJmMBt7C_u1zozx$`rorpCRcc~ev4rw{s)@!FLakIlO!ETmqx@Ate1 zlht$d`1rqttHkK~%$}~aXG8tt^9f74wQPASUgSS%a@o*T`De+5ms75Yf4lbXc;GRa zzHW_e3bz%lB)LMmTqJtL`Kxm*Qd=g@ziV|uLu9Ydq^v2u9bS4|r>pKQ($k%{L+iS# z?)^%U8=(_QGj7}T9QJs5vx_t6_U<A>zdj?5KMR}Oj`&F&ED(v6z45pGlb1GE{i*q1 zE-3k|+{1r-#gRjb^*jm@JBn9UcUv8b?iBIA_HN>tLiJeQQ#Vuv&L)4pmRe)~?8wd& zoiQ(`?d#JIn0U7GtjCdBlbbU=C;2~B6i(QEe7Z#5`%f>vGe7?H;?s=oqfh>y{(Sdx zcXGzvx>@yQl@$egg}UeU*-gK(lv?Hc&5=J+KhN~&!A&_o{9gC0WSP-4>B@l<?{{fG zV>&DlyToAWqx}pMUf$jNd2=`SgLNPOJ%0J`;LgG={dd3Lw)|UNQdreef3w$NS-9r{ zf!inMa|B<PF3w?_wnIwfwh|MUva$cl9cwifektZvf5^A5#^<cX%B3Dhwg0y?whF2_ zq%QxtyW88Op7n5@o#hT2o7?*pg#SMH^Wwpw1J(0BPmb+bIp<Ad&*twEGrkI~@^sMW z|HkLv|E;W_w@N!ikG1IT_s=D93A<Ao*H~%UB>W2Z`Ef+#J=^?khRiF?X3w9lo#1hH z<8$qVc~!wd^)r3nMHIYM)0uI7j&ih2{VE>)b4w$C6`Mtg{N%DwJ}PujZkodKPk9M^ zvgJV$tV=emIVgHSw3}T?@@Djdn50V&nM;n})V!B=V2$1X-0xpr+;@>UF#YP}YH7*( zedSZ;OnGwT$WpI}rJoe5w0?#k(9E1arOxxWmQ+~eyc=tT?BB>MyRO`uko4%YV4s-# zZ&TUrGrjNo7;?%<|8Ks!!PR^EtUXsgBvkPnKT~&~Pk3gkQQy_%nJ4!uNS@D<;X2!K zH0I>oDBb!ui5mHJi>F&pv7dHc?YZ&FPm3kWa_`mOuw3_$bK3oi=|_}j8_k=fptn-C zRyJ)?RLZxY9}&zK7d$-8ee9<E@|XR0A3r{HH*lq*o9in39h%WmNw1eU&pBlKCF#MN zCr@4!?LK(;<i+U<ANIY~5x?YUW4JxQ_lnT=C98Yym-pK{T`VxV`Fcy~*5);bJB~b@ zwoK6Npj8)t^uc+yQD2+2d*ePu{<~0O^H$&bgQ>%%nBS*Le(y`&yp%&YDCC&hgZ2%* zlD<ql^=?<i4;1k<Wosqd|5)Yory}*~<nuCOOm7V*KbMiM=Tzja>13An=FRwdJ<a1K zbLt9-D~syae%i_)^l91Ocfr-QR^=AAx88G!)_;6jb;{P`6Q<gBt$pJb_~6TnxeALH zO_2UCpmpU~!9h<mGp2n9Jrp0dS?+L+ZWc6M(4gXB`^QN6S7%<Vh<eDGiy9x?8P^n^ z5Iw;Val6Uw^f4js@5kz$%RXl0dVW|JrluCv-EN(o+B9S9ksm3CHY5lBn6S~w#yr*a zdT4BH^M4N+#Z=ZEtK+O~Luy^Oo$3nk7pVS!*Y5AmLk}c;MZH75d#bwp`>dob<=xyl zaY4x18Xx`24J*ThqP279@O_Q^cO~xK_UXE*KMS&N-i+E8Fr7W(Jm->c^(Q@VZ<=!W z@Y;t3+B{deCmC$N{j%(~?B2cC0!wPNt5@${xGq1>N$gd5%R)`vMJ;ul53?3UMnwvJ z=zg*C(+VfkBb%0sWUM@0Tl!O2l5<*B(W0b{KeE>@E-Ml3_T`%Lb<To$&u^uuePDT7 zp}n3zUv*c0V!$tFyNikQ`j011y*O`6{a?q~5$zA{oo~B%I;q)pm`qTwjsD#JcAZ+^ zw)Wt_ct_oizY8`oO;wS<+Q;=jt>yBKQ=w&VYDx>v2u%vDG|95jRTs>5sh!xmSVvf@ zb*j*6gD<T~ZiNqoOzy?xzR14(sdd}4&gfdnO;@i+oM;pXQ;0piK)mx#8>`-0L94rd z?)BII*;wvVD6E*iY(i&DWUL>*-qwG+Os^FDN@ifaD!==(2X9TWg1~OsT^rgCL@i2e zyEgNlij-x9!a*m_&2j6Ujw-4=ZQHkV=gQJuQfCSpGTVjqjq-2DS*$f=keT*r`(nm! z&Ye*<Z1<*4dX+WTJc{?(p;u+f>W^Bb()^Si!d2Sp^S-*hIN0;!)B2YIe$RvWn$9am zT`Jqu{QDwfr})drV4?rFzg;+fz{o9+M}2<5yi}j=|LfL^KdOISRWA7dx8<euf6v+( zgE-6&m&t6Kl>g-Agn68|6V5C7cRbnDX4aNfd}yIyTiz^l6WbeG9!33d{d<u&X3423 zXX-ESGTdu7XXlIh_m$HObN{VhUtaXN{_ok@NB`7M`y#nb;{Hmul{J?p|2dyy{K@ua zsr$33jk}929&VZRMu$V~Q=Y)Q3m-S!587uvuTb>I#}gmk2=M*XU3&9-*&)t9_BlU4 z^6f0himd(CBlIub&9_17)(X$Od)!CfM=xEv%!s2)C9Xm3UpM<+p4KPz|Mhu4a{t?v z|L1@A`jmg0({(>y-*bA!y#K#u?YnsPeEr|AzmophyDvIlm7Jrx_<6K&^9|whk6z;E z|Gn##)u{T;JLl6wh82rfUE5f8&FbN{O-nAkHaz<2kejmH!9Uy1D%LLTJ?Bz$hvyS_ ze^I{qJ64uoH6>!IzjeM%>*J3|-<VKTzkBk9b5mXH!$jK5IU?sA+I%LG<zG>;nCGu~ z#bSnA0{Q2=F21idAxXVkz&bkHz`4lb$=0hf3O~b^oY}PB__o!RXWIGJC%<Z(y(gv- zXu65<uD?M|mhI}4dB4P1bl*tkt4kc`keAsSB>F<}n09?*;OXtl5B<Aox7~Mp?H9*N z!5j6?_L~e9ssh;G@0DNhbVBaK`cvFo94FXz@+^(~aX)8^9ZTI4ZT%HRru|k*=i@s% zW!GwEe|m5saoWLoRZH)iTZ=r~ci`c~#i?v(oAxW!Yi7%xynpBi@0-F0F570T-Ww9X zHjUlpWbMM{h)rkLUzYy;{q>xkJpWwQ>8d{I+{bxs{`UG+edX`6+%L!HbN`(7)>q&~ z{vGwJE1sU!d+U9R_v>$=%ab;}Z>gwtiBoo+s~XDLrYwK8ZEuA^LvT}hsDf8RuS)z! z_U4+CnQHz*3Tt*xthDwF|Jt>2+FiRpy6k>084~tXKFJaL);PncN$m5C8L=K$yZRW+ zZ1;%&^R7Hwdi2)m`tx(YJ)2&?CQD_S&#%cgu_fhiA51<n+4{Fw=(F+{XIbLs{Ckzu z<x#~J`HjW4Ij--e{=*v^AN(^etUs8~x&LS8cg_mAW40?kHoX>Fcm45?*M6p_C%3<R zH{;>I*iX++#b&jayZ7|Gi(u)uPTODleD*onaFzQDYfs4DJF7R1C9?c%ed?qh8SR<U zo6ZWKTz)+G#qBNs#cr=qe=C`5_(-*;cW$Ib)qVq=pPIp|{%>UObLS~>{%XJQ{n?E@ zn!n=z`lj3F|G(J!ruE-*Ng=OoY3DzFn0R8t&ZAd$#N0DI`#?+0tarj!+l$-GF6!O= z*iq8bqjpu+EcNCQeWt_{%@LX%AFhSgdpX8>F|K&Hvga4CSEORIr-ZAb;nUfFejfO> zR&ioM22b$`0j0YM!H*cupZ;LWo5DOl&{t`-bbjuW{yBATwc>Zy@gM#>gK<Ldn}(3N zX`cIU9=yQ$ah>;{7l*f<-Po$%^7!(O2^}Av<<3^va+&XQbH$Am2LDsKVIi?)f!?fj z_uK0WvRB;;+%4^^Sp8$INJB}V;!`C<JCFKFT^<oHten-o9-c7Ycm70C=!e{cRd0;L zPfDwIBwbcL&uJd4+?JMdDe}kC1K(u$gQuCilQ;e^{8N6N<d^-G-v6b(*{|pRX20HA zn)~N}na&gSRg*eoW^Q@#xKHEACCx59s}7dS_6G$&)~{Hhu&(g`5+%DWXE)|XHye0& zGPIdp=zh|`DPw-sbNQT`{Xf3=E*E&SR=3dWZ)3Xp$HN;Z&n=$*?5e1a>!and8=vTZ zo|SE%?Dec(oS*aK3g0<8hw|UaoStF!-+Wj8r{DX}9ZULW@2ypO!{`1pUzxW?6Xl{$ zMt{=X)V1}+;%`3|)Z6~${C`3Jk@B~-XN12l;+dd&GWdh;P2=}X%SDP01PhhhzKczN zv1_6957xQ6JHGOLT$%8$rs78V#<rL0Y?sqa+jzFD7PMK(bYESsuEbOAUG@Gs;;u&j zf3ADUsl%{*?*BOPH~+7ji+%fle)6=A`!&tcEfaayZ(4joac+&;xraN<>qCu??P2iv z_P_VoB&WabC!PxYvVOcIaFXZM1>4dUe6Fmp4=Qy~`6iQ<&Yk;Pz<%rXrx!coi})SB zOcQE*9^jpL^>xMzy@lITW*>c>%Q5rE-GnP;o6KyLB&2L44*g|sl<nmTSBmhxu0Fe! z?a$GfLccCNlP$U>Xuon@ZSw93hGxw5{0BRIO>YG`Cx*)~o|yYJPsX0_E@$a~H-kH& z8@t`p_FXr(t@;1(SpV<qrx;|HExu63uHaq!X!W~YUFpL1rS^v$=A9{vmbAXOHOZ|j zd$wYw{oF669fcY<IzygLme+o`RerZv+=H(R{;P2~)@<In#(z$)^rFZLjzgmL)fYr^ zrrMi))~`$xw<~^iqUrJ9yZlV`8MB>@+-~eEIm9c?YVb+rPv4^Td+&<un^c%p>G|ZP z?9-#=`-<*x7ce~f_3wH`%H%a_5}NzAJF~^!o3mGDx_)MZk3s$1sVNi1GNGt$>KFSB MNyqmxd|+Sz0Kf!;tN;K2 literal 9967 zcmb2|=3oE=<~K3f#fx`bns@%EXvnva)HL-UvsT~ty4s!lDDhGBhPggzuSL2Yx6Mo_ zV6Z+p&-VY{EjuJz4&3leE6-|Q+36Ux-DUOH6}wdfHC8Pv>emS~e9nE@t@ZiNfA6NQ z7m87LcmJ`yrYffE-=WeFTiv;PwB_f@g!lLL`+F~c@AY)?sf#DAmwma#yD--K_C(#b zO7nV`S-qD|=brlSxPIzN{pTmvy)`@|W4TGo+GurO<e%=#C(<ilo>_E0`Sg)f$;#PF zH8p(V^h$!mgio4H(p7cKw_c{|AGm1AGNr>8=cFqopRX^uy}D0#i@{Bg&4pTrBRI-e z1}A^xExzv?=yQ0YDRbl`#q)u^PA|n~&)oJ{RH1l_rHRmsI}3v@oOu*+Sas{8m~AGK zT6)vDpE^wqy%;f>cUw%_TH)yiN^vzCJryUfIW(zN$)tLbZ+GfRw{t}YZ=K>f`?Trr zg6mC=+)kV3i7!jDSd*5~XQj7d$|KKdqWgLB4!X6aBu>A3Xv>@v*>A&FFP$v)@3E(h zr1n{(Uds>MT%{I)4`!9FJjuJ$%KX8?`hQ>N^VvTsXu7iIdc^wZf4lt!I`-ZRcvakd zFxNk4QpoEbi=R9vcNsGaYJ7KOmbt+F;#PVEYvkN)w#^rMG!Jj`4p_x~+C+9o{E2OU zGooG{Jr=cK!`hc0>g>t|Bws9BbA3k9NxrNr&rZZREj{ko)hgr7T$^x8)ZEp4d2oj7 z1(&Or!cMF*yq|M9LH$5o{@UleUs}ZOzwfsF^Gj2YvJaiYKbMtU&hEFW>D9IRe6&9M z$NRcjd2^p#Ub|$~p1rgF^oAU9f36#PWVup^%B){iT0*n!S7<rC{IglMp;emK{$bq9 zqKL!qE^cNrzEsPX!^v^qFY!|LiGUf8w+Bnq``vZTt_hm;%4KuS-;EbSHYe6;KDc-N zY2m?0yQ~u$C%hcXy5#OBt?_(%DDa_0^t)4+E=gXK+cL9x=3||#qe&|t{98SD>Nzi0 z?ibIto&VGDbMnkXyqn+KzBjJ_cXEOH*~C-&lPp{{|N8Z<5^?2SFBKu^#vE7jb;`k8 zLZ@%JnKriV&)!mfXX+7CYo~Hw=0Eu@Z0}XquTG9F^<B6uL^geMp6is}F0m5<xhGAW z{(k<_y5&VvPWZu)vlknhK7LkNzkU8=nWPufnhUxM1)@@S_p(m1IXml4sgKI6?Q-Ir zEq879UfOc?ZQ#DcoVn|FE1zGJ=%oDR`Ty&_uGc<Bw@Mu@-TCcq1HWkVd$ZrR3$4FY zEJ@w;&tfWXWc@bh+xKPHXJ6xNU2SM$%Ckj(s?MV0JFm<*w&J2##h)-1X8W7+t!oxJ zAGeg`d9YFOQ)W$5%r@gHk*SxTpPYX1`EmX9Ia+0TPhY!j?G}4FJHP+3*w(hKFF2oh zZggk7Hg)HD{_}G$d3-TZsNpy<X>YuEb@(2QkMUCTcjR?{Ulz4u-YhwX&^h0#j`Rz) zOmwljy8PaO;EfBGeGq;8PUGgK6>1YaKi=ans(xe;C1JnmxcYL@MKWuJFKuF~JT^N@ zJNU-B=dBZDJMR`YNju8)?J(1vpJ8N~YI5XWg2JciA2-hS;hpu?s<A8e*0xqP*2}Zn z=EPXfP~)s}o1z#hJG*AKJX23+jk3wJ$DVJPBFz@4%auB6M=4#G-Fk4B^u3)b7t_D- z>2155IQ#6woUqyFv$y><UGk@fd)k8)e47_|OH`&<HwB3AE4=&b%fH03^;^%$dzZeO zwg1VExaQq<j|=LOt60C=eE*P8vuoXz{QUg$;b-%rb2R+V_RG)T*Cbz`WLI)#QP4H} zSEp3}zm-<6`S-0d@XwcoukYQCUaZXix*_MI`rnUdmm2?hysc(w%*p<}xBI8h-)+0@ zR>sWq>rWr5+W&mKBFcXAKD~8sPJev9`fBOxw9tQ5Up)7{T7PTt&z09^&aaJ{JALZ5 zT`{w^`>S8SnkF9_cDZP4Y3b2=*X4^(zkKs4IkRG)&Exg|9(2Xat@F0^%{cS+XKekS z9kb??uYP=NYOMafsosC?{#k8dC!EIn`B{zKds+VZ%eDvS-gNC<csX~C){6S?7xenP zwy%DE?P)Hjj$ZT*E9c9vrXKCxH`i4B{NBv{<-z8ws`i@xjM%$N=1!)(_v7R1AD@rU zmp|6~Nc``<psblscXR%_BhLCfWQM(|_VT2__jR+)s*j#qeBsHuomDr&-^O`oeKxO= zetrE{<JBHd4zY_D4jl_%K6~}NWM)S|JG1^gAzl;JPCN70ybPxop75M=L+Z%OlRjpi z?;As=oH(rc##q%mIM;XL`|oUrHmKkG8NqYsgvmBeeS>MaM>;;d+xBe6wc-<TZ#oMZ zI#qW#-#r$5-bp)RnP_t_<M~eBTQ+UHGpFofu>Mn_bhSYuM0MT82kgw8Z5vc7?#}dJ ziTWw1SekS^H&G)(-gMoMM2Bbg6Jz-;KZ=EGuI)K)k^OvW(3+X|_Q+>O9MGIO@zxzX z^|<Xz-PYfJD($@R`qVb3_V>Mi+?GDu^fYLVVR^IS{mvI0YHF7_mLELd$17!|>o)1| zL~dW+^R1Ib(>0rUy5%wh*{$z6Y?HMpW?%fzahmdW*EOnFt39nw{W$e{r$F)FYhOIC zr%1nYX|Q4cXlBTAsOP>>kHB&@WzH#g);4jv+`d*9JnQtoSl@s9&t3bU`R0$sv2*s> zYQOn{1KiF$3Cj2^mur-_>F=4RfqM?xY-P2%{A}Kx7c%N<2WC#1xho*yWb%%=_E%Q5 z7OBRluehM}?}p&IhxT9gaa}iHp2qqoMqA09bMm7~&!EWIlWk2`n!fu82=DYPN;#1% zyYiA#yp#S77R!mVRl+Z_tY3TRPPTYK|J4I}aVkO1R=0$zoz@+-jBuIt$4&7@$XWhX z>7l;!q!vidN-}=Ky!hs%ttLEs_=Gg}rx{OPy3XD>`GfTYuHfCLex&m4yB@*uT|UQ> z$H^**<;dmjhCbJHZ)SBS_v$~N8nja4MMU5xL6N3)vCikVvoB*<|6Iq}y7Ef>#5tN` z!uAvFip}R-dCkA^{;9f|x7~x;>gM$=W3+nwj#1~rMJ~tI-`nTh;0@<0`Mgu?TH0)H z@5ug=1y`QU60SUE9rEl!>9rY)#ds~gZS;vbxa5G%#=lE;Y+1X=cCv1W%dR)C)<qxS z^?bV`YWKmMIWyL!?4EVN<?z2Wu@CWbU-G`($~~`n%ty)pO+Nc7`HvPBHU$RD4zF)2 zJ=Gx1@2xJi=|LjrfwyUgwro65Uhz`AE=;P*=0~}bf@^kNRoVxxPS*u5vcLSQx!ymm z!f{3Y-*%s<#S*O=9VL@Jb>#e}e7bmH%Y2d86Fu(B7yBqH{xFU860x&Bwf(q2bxee` zmFvW{cgs$&TsiV$@zXrxKPR@O_U)HHe(89bQOWHO!3PrOKDc>y{Y%k5i{E@*x^3w{ zJyWODzb(02M6}L^oO$=6TUsH5o44~d_rp6A@)sI-*}mFd9b>G{EMuToRqfdw;q$G{ zmTimYjp;I(_jVm})pGRD_}l&V@o~+f19I;+3i_@-6u|X;VL;QXnX`lLxjkRDsafvn zbJ?}xO|vezMdcmxHCeFKe?imK%TAqBzp{of^*yr8Zszqc{d(@hw|RfRe!dvEmF<7< zhorYvJ9Yaidh(9u<vAYJo3ZE0+^v^q$nh;ZYE!In>b}Z6pZo7N@v<M9e5=ibVTX)K z^a_K!+t02&tYW$F$d#P<ZszbiEU_0uD+5=H_;IDbmSE4GdO2b1-6y}EEYsoXbP_%g zC~@!UogJcy!rV<uH!Ng7T4T1O&vD+0+0j-BrZ;3tK5IrL<zMN4WB6%a<nNnh7i{80 z-k)8vzkT-MRi|37SGO0aTbmS`?2c?|hzT>)yAvjNX{YLfmO6ul9$JpywruF{wmIjs zAmN68&uY_&mFvUS&6{hSGx_G)r#!!B_<5F{tJ=hQWc%No$<J7KyK}@n;o2=@^!H(D z=iRwU>*vm7yWoF6SNneM<koLy+7m-Q>+XMcbfxFo#ZT1ENTjA^Zris-{pf}Mm7(0n zpVw{uyfbHo=`-^*`C`Xw{}eZ$Up1@rVO8h5$+2JSxo$k#c_1%*!h@Lx;&~fxtn<3J zSX!&h`^?9NkT?6>%i`{(|J!>|qRV(?pu~FR^exFzuPq{#1i}wB?g$d-@ILCe<cv>R zJYy&06OlVi34e_uih69Fcgfv4nJ34mem-^koAuBB#wVyxPP)N*Uqfe8Xs&z9VpCJ$ zM|qWUzf+ZG9Cpnt&@Q#8dG7n>@VpaSuSHkxUBhuk^$_dTtkTmvT&z;wyqJ>vxcGRV zcHZmy;~k#yHZx~B@TO<#>wK0DlBrpJEll*qvLGJgYtG%Wi)W^-&<U`yb(c{n)3e{W z?Hl{96;qnDXZ6nYDO0rG+}#o!+cRzRYMqkiYNiiKcR$`f+FoP#Sar|B$G6p9MxW1T z*XOKC{1hHhb}Ywb-_y26Wv=(Ctkn~3zkgW!rF6AIlk~bziuNp?Yi#Qowe~%D;BstX zr?~8lira^UrYz>XqHMH;c}k^4;%sL#O>I*Z?q5N^Y~2NG)p{=<3zgFP{A1$;-Rqf~ z{H5RITjV(exSyAPlP+ta@FK46QNrfxKaY&rf6wNZ6*#zK(QT&x8ca7Q<ZgSg=fNMw z)YDpV?cRBzwuReQ{YvpPJ@M{8&;Q)|a(2C{7YQ1sKfUv=$Q@d8(Dd+Tb{8Y-2OpV| zxt^}o;Hl(VIsM&bIU@tNor>~*jLezprR_Qtv={C^*AmAp`sj#t+KH>T|2&wG`CTgf zVe^IDPw{*+7iw&Y+S4)LktN{hiu)z2KlsJ`u~GiWoF{#z)QxGGghY1lM2(~$?Y9}v zG+8ikG%M-$>$X3+Gri=~7MpJt3jOo{dD(wc{S-IvL_OQz=XV)glzQi$*wyPKR4%c_ zDcRuHn_~SxpP1imxRZ7_^nh}`#}?grCyv$3kT1}F-d{QE!PWA|@BjUs@3fIY?cM8I z|BkZM`u{6K^yW^v``$jZ@6^>(o=;;}#;lVIudn<6vN~qJL2y95_qFd$r9ZffB<u8x zr2ZU=(<|7LUn^1bUR>?R4pw7B-D5Ks7EDwLNIUjhkL{l5*6oS0hoUbpFg&`Bu{W{p z`jmvU-FNQf7bQd;o3mq0b8p-?xwm`f%OAYq?X7NqT40HK@{`2sni>3Y>HTf1#S0@? zwZvO_BK!CJRcttSIEcA@_C(24HR%Ia>NOiyFa6!Xs{YV)SyPtBuFQM-;+G<fuQ+KW zB`j+S_RX2C)^I)lPyE^q_Ups<9=h6KzJ|H^&&!X6QvV)zANcRJGOF&l%>KmLhhEF~ z{ZITazHC-;|Aj}quWlIp5B{~kish#L^Z)M)QfpE_Tzz%o-T%jngO<wF{Qtb=N!@?p zOLgD=m+yR4<Ne~_{d;i&dWQ-X-FBqx+N*fDUXblD<LBQovfejtY^c9q{l4{4s9Ig) z)5OMPR<WyI+vXh%t&F$*d0>anuEk})FDyNzQ;=idysD_?Pve4F8Pm3_Oz?dwdhEDG zQOHA4Hq+~0-fUm6<@lB<KVGMYD*7L=y`1vqX0G0wdRBj~vzubxG6w5^+Bf@t)5qx> zXMAVhWj5E6ec_9qmqNFfNY--2dOdFLY4D5}v9h?v@l5-3T=3s3f#EykKA!pQz}S(p zUEc0ZZ9-n_Lv?$W19v_=a(GeNny1}fw9DIB^Oohc1EpmWpSa?g*L<I+QnIj%&qA&J z-><3Lk4^DkP&Z|oCCBs<ej77G-!s=|YyWYb?GX8C;*Y-yiQCU~hxjxqJT0vi^1U|k zQBG*LbnyfciNrF$N%^lLCO$hcb7t1PC=u?HKMEF<Kjh$E@VRE<rP9U+yZ7GyF32$9 z*|IMGXDhPVDz^xKzm|A`as7fmp^B**yoTi$y?PTiHMZZ+-<Pf3_}F^-+?u-^Otvg< zP}ttKKwV(VKGj|4Q=$*;71=jA^Oc90#G^%7olmysiQ6t*eJIucNo&}3|JD?@C;F8W z-#7W6TT?x4kxk1C{>4f*JT9M=mU4w(xSakl_UkErZ4bdcr><5O*4$op-Pii!v6vRQ zV@KEXc8H$~{!#6*uPQre5o=|-ua88Lr-^!gv_awt*RZ`e^uHW%nZDfY@d9=G>-MH% z*?H4A1*i0zoyouauu8af%T*pv=d|}#hIYa^f9%?3x?Ns)a+OF_&(A5zzB}e}PG#q* z*_o<%$lJ~QSbstc_t})^DSQw1-Prwbx6Bq!Z{v`8-U4j-8VWhdUMl?a#Wh@Ko?1Fd zzhU+Y-A&8n7qD*nz~yCQsphiqr&8vNSxr3FSr=K`bBiAxxpe7T6qm1$MtRaKmu*Rz z+}8unyE?q^xnOtc(bg^BMFn1Tmmco*Th8!bZ^rsJ42&#yp1<Nq$~&3qQPKD&H)GLX z@nG*3|B6&8)y7@f+qOjXIex8~m=dj|xP$kop!~sWrJi?9mL=bnIi5T1W3iX@fy8gJ zXWWxx^)@fb7Sc%gdDx-#`#CL1_xID8%k@rMwJqRcwYhHA>wQ$D{^;v%VmxYpXI8rO zWFA<*{}Bs+N2!lk-HiC|EnH5V>aPl$#WQ)ipX`+ljrk_IWxdfS)oROQ(k;#$0muI_ ztmD>QXnn_^i20T5GUu3!i6<Vo=0C1;KDg{$k&hHhYSWC%Yc4-K<r8Re*P`q015+!% zn);$Iic{N6CVakNqjOQnRp(X1v8m-;y{US#FN9yp*{WwWnfE=8iC9!p@=V-tTJcFE z`L_$W{2i~ITd;&##cH$5y_mX5i+*0TsWd%m<9sRO-g~pC6YFe0>^^-t*Z-%p0_){Y zFY9TYB`w>pdL0Y8zF5ClMqyL!dffwJ1{3DiOl`QaaaxC$j7Y(D){9(6&PeZQ`!iip zR5$p65x4GN4W|>$b~7cG9p0fYzk~Hh>Wa&*UJX+_e2%I2#h5Iyo!9z0;mXmq(&c|- zkF~S&YTqs5?BF@5E1Gnl??U{uj>rB7mOP5!nYj63x60DMU(DrBQ+oMWTN}1$*e$L2 z5y;Ygy~W<+0n0tceLvo<IK0RHz)^la{u-wptGQz@@aSvqk>|R*yJ~m%_xz6&4sE~J zKmX#x*@sp<US7#x=ePTM?$4J$uK6G3ZJ5va{*yqPU4BLFpF6L&7N)glZ{I7@zqtJU z^uknD^XYHp+UsZU75t*R@1gX9`!g8ty?yon0q^lVS=l;+Cswup<?7y5RD3=l7qwVp z>1U<YtPg4%AKdM?PT10DRMRc4S-VD3tIBJU@yUrBN>&y#*s)JprPsM4kmu#}KFvz& zCL5L`OwV@a%~5i&5s!9$E$KW{<@K(g9h!?<I9J3l-{op)Hs9P5F8*FfRO(3d?xRnZ z`yPF!a$=ci@YV>&D#P1>s-kO^j!r91`N`+xmZ4%5pCihnc)^@+kqF0E9WC*hGvlt` zUvt)Ehhua0r}gir{XU_txWQ24+zY?D6{m&RmlW+>aBrSkP2$w&QqB^d3;r{(>CRI1 zj(AfSDq?=5cKx5Cnz@oJ`aAxMJU)FiW5#EPF21AkA5^EzV%{Lc7%M-&c#Yc9wV|oK z6-6%ckII&EU(ywv6>E62^~icfN7u4ACC4{K8V8enVw!YbPde475<TtB%w-i_|KHyK zFTyFkE%~H^mh^fy#oVlf?|KuyJ+YJcZfI|JJBMdslh@<jll;_@jTVT1_TyZCXu>q^ zJyRsVE|^ilC^~2D7p8@84<Aty)nsJX5z2}W5>TJ<zaWh3!I6px=^1;%{PlHm`SilK zEEnIg{O`USZ$t%Hl?|h_{Cpa^O&HT81S(lQ_#gdD?mYE`m#sf#A?KzE3y*|f|IAVD z^s?3WQ0e#IIbzjGo)XtfJATUT(#eQ@^y=6DTh`Z>ZJYR{{6Cjyi_(*vlSV2m8g>&L z#9mcAX?T;TH#O+wG@e~2CM{+^CcWc+-QCX<+%9=cJE(Ya?KAF(%Xf`08(;qX;<$&N zYwFr(7Y-!8n)AIQZ<23sq(%3u6E}+Y9rHYY^z-TcE?J=*YppK%+e|t;#p`2JOHlS( z$2X_qtvuLPPYEx)S};G^CtXVRyu_9*r<U6ME;MS{(fPFePO<(TQRgnP08@kP4c{4) zSIx04{n2dLy>!>nGm(p`=1hGV{ZlkOuz5ye5SLP1{sT9|rBfO@ZXY^x_01nPd+w{3 zlSE#$_4hwdF4EGJoTX`9yt!XC!0U5l^=ny`B3@PPogs5VW%b2M>?ZfEkw4n@WXqS7 zoC5~0K1+NL^V>MvrPOLR!{HNm)}LQ*u%`Lr`Wt*gM}5l<O-rb8<#~LIXZ_Kr(3Rr% zxv%l6ito1UQLdSAq@<qx-}NIak`Me&_<eUx(3-xg)yImwj~y-#^0Q65@ZFdHuqq?- zp`E?AQ_oC$m*L{dfBlQhocZ(TuHQB_v}P6atFHfYbN5+CPmOTC#8bD@FY8pt$@YSu z2MT51d(R4Iwr<W;>0Vl+biz<qU1s;QDaV-fjk{JJ*|N9i(iXO7O8O49`k!L|pVHjQ zBYfQbse{DfiMIr|XwT_hznbT@0jF#zgUPY9mLdbjl&Gm2C2LA~<x=&Wm269*on7`% zn;INm8N2Y<dLhReedo%cZ3iQ3@*iobTR%??%6)3IP;HTYz|NQrAM1LGX1?SL+IU%G z`qEQREw>!oKVjAOb4fn#xrv@f-}lc;SUEMXmD{z`cCJn6xdUhHK0n-YGjF|%(1(+8 zQ>8<n3C+I#A*vv1#@?1`VVfVg%n*te?vfVh_ejoK{Cus&w&yy2DrQYrKUl4<5!$S~ zsOD74^2d$dDr$WO7uemL56(S*wB6jrLSwZ=8t?x}AGgjD&waDfRIxDWdG&h^yGn+q zk3J;ShaS3mXiKHuA#Uq`JYl<5T)N!Q{?AR%gJt!M<NjPT{XA<E^WQqkq@H=Wqa%rX zlkB3&3mukvv>dGXxZ-?Uu$#osn`@hUoGzYLS?PAOW5tt28`eD6eRb(k=jx)^OP}Vt zH^#C?wr@MT*rI33zO^$Hw(ivyfBZz_@Bw+v43{@8f;{dQuQuMT*U;axXu%&(e%(V; zPW>+E@z#;~bey57)q47_?JJV^O!fGf@>cG&O!=j&3wmOiF4~yo#~%Bh>^P@;&WWjc zFJJ#)61_R+dXLK;_sMf-s{9O*Gh*CUUekH-mR^6Iy@6i8UjNPrpMU#p?pyD*XuI}y z`Ooy}?d;Emu7CPd6p-v6u>7;wk)MfgB8_EqPPe^Zb8d!bgQkWjZ_l4~yOZ<`crLnf z1#A7|W|(=ewz{sYknxV`y;_UA_7>)S-!}fPvtC}`ujkkQL4UX4i7#55B1zIeg%e*@ zCHtK@AQax56z$@WnDR_hbi1hPl)7~fRqh;|-oMgPeXGdIil_4pHW(aYS@FiCzPPY+ zS;CF;>g?0o(|?PnERvriFB>CKalW{2Q@dd0WZMaa-}nw!#W|UBzIb}@_TkHmw^tu$ z+bgOyooVO0-<4MS8)7#(gvpA^8T{2&t4ZoAZ{}Y-qcLRm+4HAG4VFB+@mbX1esyqA z{Y>9?5e0A6bRw?IQI3|WU&f<<ZfWJOV!J4jpInc7gbvD0Q&|2fFQZSUJSu{9$%Zut zMGuH}vnxs7jD8T4bm^gO$?=<-_tFlmvHPF<{>zK|E)oZ(Z~bCs(dxDGx%je4X-^V( zckS}XIy7z3AFk4}f8R7LHBWYSdQaWp_0;)G$fxxU!I$hb|4#ZAYO-dN&BjR8x*PUx zT|w0cj~sc<ETh}^E!!&3c>2DbNkTSe|JYY2&Yq@cTDkJ!!7mbhGn*S_)r?N3$gDnY zG`ZT*Xx>!|ky(P>J11Fhjrw=wu)~EvZu;dK^*Zx?&!q=FbvJx-YtP2wn8U(4`#<V+ zx|=22s5-_3S^u&+qq6nHo23R<>@O}zyi@o1P5<RD$M1bCeE4p~6@kS{SLN?a(aqgh zdP&jmp<I<=L0M5z$<E7<9zJ<t%~Wx}Y#R4Vg?Te>uTabCj=r?@+WYF`@(VI7X6|`g zR<gBw&EbwC54SB7bUSF(bzbS<yr{3u+P!g~BLAHzv3aX+{lV1XQq1pDCG7iBH!tN7 z4hm6Id(gh2SJI!Er{3+V_<<sxrfe-?`yY#Z{#KNJJ?8i8=og(8i(A7x8(z71J(<k0 zo8e8=T+S`+v8GyEq*gI8#~!@IaH0L|Bzbp<4s%@*Ne1g}nu@$Noy^kSycs{Qr+K_& zPF*2!C8=QTr>zV^pO$@omYkk!R&H^7>%WlO8jnw_PWd`bQ9ECA+a__>#>>mgn|ivO z^8Prl2w*?Vs($kZ#}78;))wh!55#hL-L5n+O%izi;dIXn^<AZba{>&HKRzJOzTuc+ zxB_!RF1M7fw71S3Y5to>7jIQPxck=385y~9@7B!aOu8oacrxeVKG%oN$GAS;nGzZs zURo;qPr0yVO6Y@iRd2rpeigl`rR`+v_~x(q?=R1_nv+dJbpr3IPZjuizI#oE4zHS9 zQ_$AWhI^kJTz1PdXYH0kt1IbGlhac#L~fii_3&HK^y;Thv64x)%O_Z=tIM7a;^W`$ zzWA*{h`z_6J8|2uzb^TDt}6WE$E8o--t#U^&gR|}uwR+SH&amci#JEO@YzeJCN;#n z_)m2gU8Ok9cCmwZtmF!9`I8<Su1ppd?LHd+I(F;o)e$*C3)Vbm`D$}r&A3rr<M{2$ z+6wE7wcO2@<Ue-ueXcQ@xuWOWmdE_oN33}MOWsuT5;FSexTMiW^2RLtS6@sDx7g0< z`o)yx_=bII=n9peE6PHDm@Cgw3tP8I_<478^0WzSPAv|6oN3t^q5P^_M<#XV1&uXR zB3dWvix~GYPharyS(bAA?5X-&O#SajotUt;@t6}|%Lb;GQEYoXo|x$rWX`?tvUK;Z zZ_iJDWbHh>+osv)$*EUGRRt?~&DKn8yDZPFvEtv6xvp6+W;-|DczdGNl;xTbukn_5 zZ`3ZHILO2)<~8GMm8ce<qNe$mwQF_P-oBjB#_Xd$r)py5?T?2ddl?^Wn6f{XH^S${ z)klUO)=dfVf1Q_QI)i)l?cO;Z;#o5+yBT69O6<BSznI<l(dqrmoGs^iTXNX7XLwql z<a_7Nu9A0Y@-mfw+uttiZ%pmEE8%DNQ0epx(f_gcHIDqhR=T(I|6h@p!u6jIGibKV zPPFE`ebW3>iO2nx+zsgy)m@4rm8G4xT0T6%;cR|ZdZt`XRH5#lHUD0)nXNQZ{d|8* zo=-kQy56tqiK%C1{g;-$d*{#n8+US^{hz$Vf5XuqZl)ejC(nCQpEP@7{>gRLM(cU* z%stGxd~Qm*qrt@T<_|4>?SHp@I{UF_^@0B5ZRza=C-Swune)x(9Qwa_oBaDp+{?F_ zCY;xJSfAG7f2Tj^ii-YS?j!GSFI~FKh@(p-u0iczH~U_m)+hhZ>we_^w=4he|L*lE z|2De|e!RZu^on`^e_82#|M_$M{#T}-|7!}>iby4EOV-|>C3R%pj`r|f72nG9-#A>X zo?URNoq0jus;J|)V&At}pE9aG7`JiHhO1r24ff|2yToTMHWrjOZ~k*=@;U3Z)r^gH zc5_@$+O68N_VMTAkvcQ(yffKSta`CuwDlpcli2bPEAqt}_y3;lI%yx<99N#)D~HoB zzSz#S<i<&*Bj>(ZotYpbP`TQe-D97q=H@r~XSVTe{dwwjUXiWW=5iLVklvg(TQu7I z*QQTB<9xAcL96ZKw<3=v9u`jxTg7C1q~hs$gRGzDd~VoVKi~GUtnZ%od8X<o?q+N* z{I4GD{mi^YW|Qt8|4KJT#+wS$7n*SYpWT1$_ksSJr;?%To&_F%@{`+e<6_fPUpX18 zGe$R@7k>F2HvP~YaSNNWEoT;F{?Pt3ebcjve{_59ADGADm(soSR`J)PQDp}v-$}RX zP`&-fX!pkY{pTW2Jn(;VtH^2Ii`fZ(E7!iNc)qU8^zYTa3Hdkgi8TH@RvXkCb??ok zeM{~J?AJHc)&A|*aozoiY0#C(ps5KK-vV=Q^tU@(s8)otE@HXi@kzAa;>`3pX`fUW z5ABkkxqWg;=z59ln{N-_FM2SAkKx#C_j759@*SGW8HW9xH{Di+m9T7h`Ec)t_*2>I zw6D#pe^s{4?tikc^2U@)^Pjw2v^$%9zRJ9NH@B}a+V7mN_Tv%HwON`<FC|lMmcNkv zQsh(5dAzUfxBrp9ZTE%hK3{%U{-9Fg`@&<q*HvTUIgU>*fBA04-G8y4o}7x!YAbio z>3J8y&~Kf#uk`usbF$Z!?=P%8AsctrOuK2#b}=LOV|`P7cyHe8d~*4+=9k=C^=I{7 z?f%wtt*3%BJ~>^k@1A*?s(rfGSNm@lK6W(bt@x+@WiRVAL+Agi^UKUXeUsnYd)Dp$ zXAw@<+d=m~ewcWoAanN0!kq`Ajha`_xMAXSZM)0uT`grV``nKxr|1OUeK<*H_77JD zo=O32R>6C_yfjyw)?_fXnHsp?(NH_ZKya~!%CbK~^>!QTvOQE}qz=m%wRo7{m|}6@ zvr&yd^UVY4A(KzMIu<ScGdVqA_o?u8_m3CYr!k}`TPuVvkGzy$W>Dm`=e3aiuMPRd zIS;uX{?O4=F{*jF%~`-Iv;C)|O;i%Y{i)GwR_uDSXd2g_ef$rvh3;MbEl!W~=Hm5B z8I~0DXo`1!bo@0r#O09FbxF%WF86&Od)=n3Zp!DrmOOu_*F6hIk=c5ceifRgNfBK> zR^0aDH|9M~JT>Xs-{fcYO8*a+Hvik6@iPAKzvrO`|2-Gg;q3iW-{kVd-qy>d&nN0b zp}R=ZOA%LbS(m1l^B;6ngoFrO-%%eVD1YnOk!@=eXDoGLaF%}IT6BP=Z}zJt!RKUL zYJM#X<}kZH&0^`l1BWO7czEOFxt-phUvW)S`gnO>#HZ)3XSdEbRQ<faeR)g89YsH} zhvDUopkhqFdiKx1@!wO8|Ib&MYL&Ar|Fc?O*-VfAw@-9`nnfPH^(C;n=EA&xE%h(N zKMMXXd)8SM$aZ4V6U`sn-pnjN7~Ey~K(lkV-1l|HC3i3I{%JZFd!(xUM~FeWoz2_b z5r@ABH^1B<<;-^LD#yH##`nVF_PbPszweDd*R3@3f6etTvN{aubN|PQzWHw+XZG#C z|Kw?%_iMVNTPE_Z-?aLI;@ldwa}RfzhZ-N-!{G7l|JrMtoc_9>cq{PB`tg#$NuF01 zZA(}1xsqWJR_dVgO(v^cI`_AL{nqPGFLul?=6CoqO{nd;hj-%D*BMXr7H>~!Kl(bC zbLNk`30KNCo7pN!NZCjn`pe!Z+s_@Y6ybYaeRiwZpQAH{eqDGLTXaj%e&xE_<lPet z4Vn25cKVv$3UW>imtj0H_iMh4J>Ol;(*JG-cS1MLc2C=9ZeUyU|KYKI_UoqvWS1?z zP}Z*CUHfSDyIo!B!uF;1haBdeDT|(IadB&sTUYjM#Y+3RUraj+HEwi<JY6iW{cx-N zZn1k0zApT~jKi^J^UgK?b9$o}MOJVe60NVkAd)lH<g<Qd8oyofs}oI+|L*<HRG%^1 z*~sn2zLHbC(yRubRQ{}6w0{3Rp?#AIvnoBGyo`N%(|lji9qs~#N5B3}ugFQbT(Bqo z?AfIY)IN8(|G2U7$rG9LkG3*wXx#427JF~bUYY6onGHS$^>e2#@$}6Knmp^u5;fD6 Sp8K7C$y-*<3}N`dzyJU~P@B{M diff --git a/.docker/docker-compose.yml b/.docker/docker-compose.yml index 67204a64ca..54b7b8edbb 100644 --- a/.docker/docker-compose.yml +++ b/.docker/docker-compose.yml @@ -277,6 +277,7 @@ services: image: registry.datalab.tuwien.ac.at/dbrepo/ui:1.4.5 environment: NUXT_PUBLIC_API_CLIENT: "${BASE_URL:-http://localhost}" + NUXT_PUBLIC_API_SERVER: "${BASE_URL:-http://localhost}" NUXT_PUBLIC_UPLOAD_CLIENT: "${BASE_URL:-http://localhost}/api/upload/files" depends_on: dbrepo-search-service: diff --git a/.docs/.swagger/api.yaml b/.docs/.swagger/api.yaml index b24f1e6f93..c5c2b5ee8d 100644 --- a/.docs/.swagger/api.yaml +++ b/.docs/.swagger/api.yaml @@ -8374,14 +8374,14 @@ components: type: string resumptionToken: type: string - parametersString: + untilDate: type: string + format: date-time fromDate: type: string format: date-time - untilDate: + parametersString: type: string - format: date-time BannerMessageDto: required: - id diff --git a/.docs/api/analyse-service.md b/.docs/api/analyse-service.md index 3f5921968c..3529f0f3df 100644 --- a/.docs/api/analyse-service.md +++ b/.docs/api/analyse-service.md @@ -31,8 +31,11 @@ the [Storage Service](../storage-service), analysis for data types and primary k with [`csv.Sniff().sniff(...)`](https://docs.python.org/3/library/csv.html#csv.Sniffer). This step is optional when the separator was provided via HTTP-payload: `{"separator": ";", ...}` 3. With the separator known (either from step 2 or via HTTP-payload), the [`Pandas`](https://pypi.org/project/pandas/) - guesses the headers and column types and enums, if the HTTP-payload contains `{"enum": true, ...}`. The data type - is guessed by a combination of Pandas and heuristics. + guesses the headers and column types and enums by analysing the first 10.000 rows, if the HTTP-payload contains + `{"enum": true, ...}`. The data type is guessed by a combination of Pandas and heuristics. + +If your datasets are larger than 10.000 rows, increase the number of lines analysed by setting the `ANALYSE_NROWS` +variable to the desired integer. ### Examples diff --git a/.docs/api/gateway-service.md b/.docs/api/gateway-service.md index 923b95a9f3..26ad76f092 100644 --- a/.docs/api/gateway-service.md +++ b/.docs/api/gateway-service.md @@ -39,6 +39,12 @@ services: If your TLS private key as a password, you need to specify it in the `dbrepo.conf` file. +### Connection Timeouts + +The reverse proxy has a defined timeout of 90 seconds on all requests (these are also enforced in the +[User Interface](../ui) using the [`axios`](https://www.npmjs.com/package/axios) module). For large databases these +timeouts may need to be increased, e.g. the timeout for creating subsets is by default already increased to 600 seconds. + ### User Interface To serve the [User Interface](../ui/) under different port than `80`, change the port mapping in diff --git a/.docs/api/metadata-service.md b/.docs/api/metadata-service.md index 039f2fa097..9d9c39ee25 100644 --- a/.docs/api/metadata-service.md +++ b/.docs/api/metadata-service.md @@ -23,9 +23,21 @@ types), semantic concepts (i.e. ontologies) and relational metadata (databases, ## Generation -Most of the metadata available in DBRepo is generated automatically, leveraging the available information and taking -the burden away from researchers, data stewards, etc. For example, the schema (names, constraints, data length) of -generated tables and views is obtained from the `information_schema` database maintained by MariaDB internally. +DBRepo generates metadata for managed tables automatically by querying MariaDB's internal structures +(e.g. `information_schema`). + +!!! info "Managed Tables" + + DBRepo only manages system-versioned tables, other tables are not supported. These other tables are ignored by + DBRepo and thus can co-exist in the same database. If you want a non-system-versioned table `my_table` to be managed + by DBRepo, make it system-versioned: + + ```sql + ALTER TABLE `my_table` ADD SYSTEM VERSIONING; + ``` + + Then, refresh the managed table index by navigating to your database > Settings > Schema > Refresh. This action can + only be performed by the database owner. ## Identifiers diff --git a/.docs/examples/health.md b/.docs/examples/health.md new file mode 100644 index 0000000000..ebd2673c44 --- /dev/null +++ b/.docs/examples/health.md @@ -0,0 +1,25 @@ +--- +author: Martin Weise +--- + +## tl;dr + +[:fontawesome-solid-database: Dataset](https://dbrepo1.ec.tuwien.ac.at/database/32/info){ .md-button .md-button--primary target="_blank" } + +## Description + +This dataset contains anonymous behavioural and ocular recordings from healthy participants during performance of a +decision-making task between action movements in which we studied the influence of social facilitation and social +pressure. The original dataset is available on [Kaggle](https://doi.org/10.34740/kaggle/ds/4292829). + +## Solution + +Importing the original dataset from Kaggle into DBRepo allows for exploratory analysis and increased value for secondary +use, e.g. with our [`pandas`](https://pypi.org/project/pandas/)-compatible [Python Library](../../api/python). + +## DBRepo Features + +- [x] System versioning +- [x] Subset exploration +- [x] Large dataset (3 tables, around 6GB) +- [x] External data access for analysis diff --git a/dbrepo-analyse-service/app.py b/dbrepo-analyse-service/app.py index 2dc9161746..def401c0e2 100644 --- a/dbrepo-analyse-service/app.py +++ b/dbrepo-analyse-service/app.py @@ -225,6 +225,7 @@ app.config["S3_ACCESS_KEY_ID"] = os.getenv('S3_ACCESS_KEY_ID', 'seaweedfsadmin') app.config["S3_BUCKET"] = os.getenv('S3_BUCKET', 'dbrepo') app.config["S3_ENDPOINT"] = os.getenv('S3_ENDPOINT', 'http://localhost:9000') app.config["S3_SECRET_ACCESS_KEY"] = os.getenv('S3_SECRET_ACCESS_KEY', 'seaweedfsadmin') +app.config["ANALYSE_NROWS"] = int(os.getenv('ANALYSE_NROWS', '10000')) app.json_encoder = LazyJSONEncoder diff --git a/dbrepo-analyse-service/determine_dt.py b/dbrepo-analyse-service/determine_dt.py index a0890c2b7a..462f8b652b 100644 --- a/dbrepo-analyse-service/determine_dt.py +++ b/dbrepo-analyse-service/determine_dt.py @@ -10,7 +10,7 @@ import pandas from numpy import dtype, max, min from flask import current_app from pandas import DataFrame -from pandas.errors import EmptyDataError +from pandas.errors import EmptyDataError, ParserError from api.dto import ColumnAnalysisDto, DataTypeDto, AnalysisDto from clients.s3_client import S3Client @@ -45,12 +45,14 @@ def determine_datatypes(filename, enum=False, enum_tol=0.0001, separator=',') -> for encoding in ['utf-8', 'cp1252', 'latin1', 'iso-8859-1']: try: logging.debug(f"attempt parsing .csv using encoding {encoding}") - df = pandas.read_csv(fh, delimiter=separator, nrows=100, lineterminator=line_terminator, - index_col=False, encoding=encoding) + df = pandas.read_csv(fh, delimiter=separator, nrows=current_app.config['ANALYSE_NROWS'], + lineterminator=line_terminator, index_col=False, encoding=encoding) logging.debug(f"parsing .csv using encoding {encoding} was successful") break + except ParserError as error: + raise IOError(f"Failed to parse .csv using separator {separator}: {error}") except (UnicodeDecodeError, EmptyDataError) as error: - logging.warning(f"Failed to parse .csv using encoding {encoding}: {error}") + raise IOError(f"Failed to parse .csv using encoding {encoding}: {error}") if df is None: raise IOError( f"Failed to parse .csv: no supported encoding found (one of: utf-8, cp1252, latin1, iso-8859-1)") diff --git a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java index 8f5f4a8214..a7c7494692 100644 --- a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java +++ b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/SubsetEndpoint.java @@ -86,19 +86,11 @@ public class SubsetEndpoint { schema = @Schema(implementation = ApiErrorDto.class))}), }) public ResponseEntity<List<QueryDto>> list(@NotNull @PathVariable("databaseId") Long databaseId, - @RequestParam(name = "persisted", required = false) Boolean filterPersisted, - Principal principal) + @RequestParam(name = "persisted", required = false) Boolean filterPersisted) throws DatabaseUnavailableException, DatabaseNotFoundException, RemoteUnavailableException, QueryNotFoundException, NotAllowedException, MetadataServiceException { log.debug("endpoint find subsets in database, databaseId={}, filterPersisted={}", databaseId, filterPersisted); final PrivilegedDatabaseDto database = metadataServiceGateway.getDatabaseById(databaseId); - if (!database.getIsPublic()) { - if (principal == null) { - log.error("Failed to find subsets in database: no access"); - throw new NotAllowedException("Failed to find subsets in database: no access"); - } - metadataServiceGateway.getAccess(databaseId, UserUtil.getId(principal)); - } final List<QueryDto> queries; try { queries = subsetService.findAll(database, filterPersisted); @@ -151,8 +143,7 @@ public class SubsetEndpoint { public ResponseEntity<?> findById(@NotNull @PathVariable("databaseId") Long databaseId, @NotNull @PathVariable("subsetId") Long subsetId, @NotNull HttpServletRequest httpServletRequest, - @RequestParam(required = false) Instant timestamp, - Principal principal) + @RequestParam(required = false) Instant timestamp) throws DatabaseUnavailableException, DatabaseNotFoundException, RemoteUnavailableException, QueryNotFoundException, FormatNotAvailableException, StorageUnavailableException, QueryMalformedException, SidecarExportException, StorageNotFoundException, NotAllowedException, UserNotFoundException, @@ -161,13 +152,6 @@ public class SubsetEndpoint { log.debug("endpoint find subset in database, databaseId={}, subsetId={}, accept={}, timestamp={}", databaseId, subsetId, accept, timestamp); final PrivilegedDatabaseDto database = metadataServiceGateway.getDatabaseById(databaseId); - if (!database.getIsPublic()) { - if (principal == null) { - log.error("Failed to find subsets in database: no access"); - throw new NotAllowedException("Failed to find subsets in database: no access"); - } - metadataServiceGateway.getAccess(databaseId, UserUtil.getId(principal)); - } final QueryDto query; try { query = subsetService.findById(database, subsetId); @@ -262,7 +246,7 @@ public class SubsetEndpoint { StorageNotFoundException, QueryStoreInsertException, TableMalformedException, PaginationException, QueryNotSupportedException, NotAllowedException, UserNotFoundException, MetadataServiceException { log.debug("endpoint create subset in database, databaseId={}, data.statement={}, principal.name={}, " + - "page={}, size={}, timestamp={}", databaseId, data.getStatement(), principal.getName(), page, size, + "page={}, size={}, timestamp={}", databaseId, data.getStatement(), principal.getName(), page, size, timestamp); /* check */ endpointValidator.validateDataParams(page, size); @@ -360,21 +344,21 @@ public class SubsetEndpoint { } try { final QueryDto query = subsetService.findById(database, subsetId); - final Long count = subsetService.reExecuteCount(database, query); final HttpHeaders headers = new HttpHeaders(); - headers.set("X-Count", "" + count); headers.set("Access-Control-Expose-Headers", "X-Count"); - if (request.getMethod().equals("GET")) { - final QueryResultDto result = subsetService.reExecute(database, query, page, size, null, null); - result.setId(subsetId); - log.trace("re-execute query resulted in result {}", result); + if (request.getMethod().equals("HEAD")) { + final Long count = subsetService.reExecuteCount(database, query); + headers.set("X-Count", "" + count); return ResponseEntity.ok() .headers(headers) - .body(result); + .build(); } + final QueryResultDto result = subsetService.reExecute(database, query, page, size, null, null); + result.setId(subsetId); + log.trace("re-execute query resulted in result {}", result); return ResponseEntity.ok() .headers(headers) - .build(); + .body(result); } catch (SQLException e) { log.error("Failed to establish connection to database: {}", e.getMessage()); throw new DatabaseUnavailableException("Failed to establish connection to database: " + e.getMessage(), e); diff --git a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java index f4d31bbc21..e26a37b43a 100644 --- a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java +++ b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/TableEndpoint.java @@ -25,6 +25,7 @@ import io.swagger.v3.oas.annotations.media.Schema; import io.swagger.v3.oas.annotations.responses.ApiResponse; import io.swagger.v3.oas.annotations.responses.ApiResponses; import io.swagger.v3.oas.annotations.security.SecurityRequirement; +import jakarta.servlet.http.HttpServletRequest; import jakarta.validation.Valid; import jakarta.validation.constraints.NotBlank; import jakarta.validation.constraints.NotNull; @@ -32,6 +33,7 @@ import lombok.extern.log4j.Log4j2; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.core.io.InputStreamResource; import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; import org.springframework.http.HttpStatus; import org.springframework.http.ResponseEntity; import org.springframework.security.access.prepost.PreAuthorize; @@ -189,6 +191,7 @@ public class TableEndpoint { @RequestParam(required = false) Instant timestamp, @RequestParam(required = false) Long page, @RequestParam(required = false) Long size, + @NotNull HttpServletRequest request, Principal principal) throws DatabaseUnavailableException, RemoteUnavailableException, TableNotFoundException, TableMalformedException, PaginationException, QueryMalformedException, MetadataServiceException, @@ -221,7 +224,12 @@ public class TableEndpoint { final HttpHeaders headers = new HttpHeaders(); headers.set("Access-Control-Expose-Headers", "X-Count"); try { - headers.set("X-Count", "" + tableService.getCount(table, timestamp)); + if (request.getMethod().equals("HEAD")) { + headers.set("X-Count", "" + tableService.getCount(table, timestamp)); + return ResponseEntity.ok() + .headers(headers) + .build(); + } final QueryResultDto dto = tableService.getData(table, timestamp, page, size); return ResponseEntity.ok() .headers(headers) diff --git a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java index e9dbfd5c3a..e4ddab6036 100644 --- a/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java +++ b/dbrepo-data-service/rest-service/src/main/java/at/tuwien/endpoints/ViewEndpoint.java @@ -92,8 +92,7 @@ public class ViewEndpoint { }) public ResponseEntity<List<ViewDto>> getSchema(@NotBlank @PathVariable("databaseId") Long databaseId) throws DatabaseUnavailableException, DatabaseNotFoundException, RemoteUnavailableException, - ViewMalformedException, ViewNotFoundException, DatabaseMalformedException, ViewSchemaException, - MetadataServiceException { + ViewNotFoundException, DatabaseMalformedException, MetadataServiceException { log.debug("endpoint inspect view schemas, databaseId={}", databaseId); final PrivilegedDatabaseDto database = metadataServiceGateway.getDatabaseById(databaseId); try { @@ -265,20 +264,19 @@ public class ViewEndpoint { metadataServiceGateway.getAccess(databaseId, UserUtil.getId(principal)); } try { - final Long count = viewService.count(view, timestamp); final HttpHeaders headers = new HttpHeaders(); - headers.set("X-Count", "" + count); headers.set("Access-Control-Expose-Headers", "X-Count"); - if (request.getMethod().equals("GET")) { - final QueryResultDto result = viewService.data(view, timestamp, page, size); - log.trace("get view data resulted in result {}", result); + if (request.getMethod().equals("HEAD")) { + headers.set("X-Count", "" + viewService.count(view, timestamp)); return ResponseEntity.ok() .headers(headers) - .body(result); + .build(); } + final QueryResultDto result = viewService.data(view, timestamp, page, size); + log.trace("get view data resulted in result {}", result); return ResponseEntity.ok() .headers(headers) - .build(); + .body(result); } catch (SQLException e) { log.error("Failed to establish connection to database: {}", e.getMessage()); throw new DatabaseUnavailableException("Failed to establish connection to database: " + e.getMessage(), e); diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/SubsetEndpointUnitTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/SubsetEndpointUnitTest.java index d554667467..6a0015b6f6 100644 --- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/SubsetEndpointUnitTest.java +++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/SubsetEndpointUnitTest.java @@ -29,7 +29,6 @@ import org.springframework.security.test.context.support.WithMockUser; import org.springframework.test.context.junit.jupiter.SpringExtension; import java.io.InputStream; -import java.security.Principal; import java.sql.SQLException; import java.time.Instant; import java.util.List; @@ -69,7 +68,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest { DatabaseNotFoundException, RemoteUnavailableException, SQLException, MetadataServiceException { /* test */ - final List<QueryDto> response = generic_findAllById(DATABASE_3_ID, DATABASE_3_PRIVILEGED_DTO, null); + final List<QueryDto> response = generic_findAllById(DATABASE_3_ID, DATABASE_3_PRIVILEGED_DTO); assertEquals(6, response.size()); } @@ -79,17 +78,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest { /* test */ assertThrows(DatabaseNotFoundException.class, () -> { - generic_findAllById(null, null, null); - }); - } - - @Test - @WithAnonymousUser - public void findAllById_privateNoAccess_fails() { - - /* test */ - assertThrows(NotAllowedException.class, () -> { - generic_findAllById(DATABASE_1_ID, DATABASE_1_PRIVILEGED_DTO, null); + generic_findAllById(null, null); }); } @@ -105,7 +94,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest { .thenReturn(DATABASE_3_PRIVILEGED_DTO); /* test */ - generic_findById(QUERY_5_ID, QUERY_5_DTO, MediaType.APPLICATION_JSON, null, null); + generic_findById(QUERY_5_ID, QUERY_5_DTO, MediaType.APPLICATION_JSON, null); } @Test @@ -126,7 +115,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest { .thenReturn(mock); /* test */ - generic_findById(QUERY_5_ID, QUERY_5_DTO, MediaType.parseMediaType("text/csv"), null, null); + generic_findById(QUERY_5_ID, QUERY_5_DTO, MediaType.parseMediaType("text/csv"), null); } @Test @@ -147,7 +136,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest { .thenReturn(mock); /* test */ - generic_findById(QUERY_5_ID, QUERY_5_DTO, MediaType.parseMediaType("text/csv"), Instant.now(), null); + generic_findById(QUERY_5_ID, QUERY_5_DTO, MediaType.parseMediaType("text/csv"), Instant.now()); } @Test @@ -161,7 +150,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest { /* test */ assertThrows(DatabaseNotFoundException.class, () -> { - generic_findById(QUERY_5_ID, QUERY_5_DTO, MediaType.APPLICATION_JSON, null, null); + generic_findById(QUERY_5_ID, QUERY_5_DTO, MediaType.APPLICATION_JSON, null); }); } @@ -272,9 +261,6 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest { /* test */ final ResponseEntity<QueryResultDto> response = subsetEndpoint.getData(DATABASE_3_ID, QUERY_5_ID, null, httpServletRequest, null, null); assertEquals(HttpStatus.OK, response.getStatusCode()); - assertNotNull(response.getHeaders().get("X-Count")); - assertEquals(1, response.getHeaders().get("X-Count").size()); - assertEquals(QUERY_5_RESULT_NUMBER, Long.parseLong(response.getHeaders().get("X-Count").get(0))); assertNotNull(response.getHeaders().get("Access-Control-Expose-Headers")); assertEquals(1, response.getHeaders().get("Access-Control-Expose-Headers").size()); assertEquals("X-Count", response.getHeaders().get("Access-Control-Expose-Headers").get(0)); @@ -325,9 +311,6 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest { /* test */ final ResponseEntity<QueryResultDto> response = subsetEndpoint.getData(DATABASE_1_ID, QUERY_1_ID, USER_1_PRINCIPAL, httpServletRequest, null, null); assertEquals(HttpStatus.OK, response.getStatusCode()); - assertNotNull(response.getHeaders().get("X-Count")); - assertEquals(1, response.getHeaders().get("X-Count").size()); - assertEquals(QUERY_1_RESULT_NUMBER, Long.parseLong(response.getHeaders().get("X-Count").get(0))); assertNotNull(response.getBody()); } @@ -464,7 +447,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest { }); } - protected List<QueryDto> generic_findAllById(Long databaseId, PrivilegedDatabaseDto database, Principal principal) + protected List<QueryDto> generic_findAllById(Long databaseId, PrivilegedDatabaseDto database) throws DatabaseUnavailableException, NotAllowedException, QueryNotFoundException, DatabaseNotFoundException, RemoteUnavailableException, SQLException, MetadataServiceException { @@ -481,16 +464,16 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest { } /* test */ - final ResponseEntity<List<QueryDto>> response = subsetEndpoint.list(databaseId, null, principal); + final ResponseEntity<List<QueryDto>> response = subsetEndpoint.list(databaseId, null); assertEquals(HttpStatus.OK, response.getStatusCode()); return response.getBody(); } - protected void generic_findById(Long subsetId, QueryDto subset, MediaType accept, Instant timestamp, - Principal principal) throws UserNotFoundException, DatabaseUnavailableException, - StorageUnavailableException, NotAllowedException, QueryMalformedException, QueryNotFoundException, - DatabaseNotFoundException, SidecarExportException, RemoteUnavailableException, FormatNotAvailableException, - StorageNotFoundException, SQLException, MetadataServiceException { + protected void generic_findById(Long subsetId, QueryDto subset, MediaType accept, Instant timestamp) + throws UserNotFoundException, DatabaseUnavailableException, StorageUnavailableException, + NotAllowedException, QueryMalformedException, QueryNotFoundException, DatabaseNotFoundException, + SidecarExportException, RemoteUnavailableException, FormatNotAvailableException, StorageNotFoundException, + SQLException, MetadataServiceException { /* mock */ when(queryService.findById(DATABASE_3_PRIVILEGED_DTO, subsetId)) @@ -499,7 +482,7 @@ public class SubsetEndpointUnitTest extends AbstractUnitTest { .thenReturn(accept.toString()); /* test */ - final ResponseEntity<?> response = subsetEndpoint.findById(DATABASE_3_ID, subsetId, mockHttpServletRequest, timestamp, principal); + final ResponseEntity<?> response = subsetEndpoint.findById(DATABASE_3_ID, subsetId, mockHttpServletRequest, timestamp); assertEquals(HttpStatus.OK, response.getStatusCode()); assertNotNull(response.getBody()); } diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/TableEndpointUnitTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/TableEndpointUnitTest.java index 29a33d1d9c..76c34ef47c 100644 --- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/TableEndpointUnitTest.java +++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/TableEndpointUnitTest.java @@ -9,6 +9,7 @@ import at.tuwien.exception.*; import at.tuwien.gateway.MetadataServiceGateway; import at.tuwien.service.TableService; import at.tuwien.test.AbstractUnitTest; +import jakarta.servlet.http.HttpServletRequest; import lombok.extern.log4j.Log4j2; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -40,6 +41,9 @@ public class TableEndpointUnitTest extends AbstractUnitTest { @Autowired private TableEndpoint tableEndpoint; + @Autowired + private HttpServletRequest httpServletRequest; + @MockBean private TableService tableService; @@ -152,11 +156,8 @@ public class TableEndpointUnitTest extends AbstractUnitTest { .thenReturn(TABLE_8_DATA_DTO); /* test */ - final ResponseEntity<QueryResultDto> response = tableEndpoint.getData(DATABASE_3_ID, TABLE_8_ID, null, null, null, null); + final ResponseEntity<QueryResultDto> response = tableEndpoint.getData(DATABASE_3_ID, TABLE_8_ID, null, null, null, httpServletRequest, null); assertEquals(HttpStatus.OK, response.getStatusCode()); - assertNotNull(response.getHeaders().get("X-Count")); - assertEquals(1, response.getHeaders().get("X-Count").size()); - assertEquals(QUERY_5_RESULT_NUMBER, Long.parseLong(response.getHeaders().get("X-Count").get(0))); assertNotNull(response.getHeaders().get("Access-Control-Expose-Headers")); assertEquals(1, response.getHeaders().get("Access-Control-Expose-Headers").size()); assertEquals("X-Count", response.getHeaders().get("Access-Control-Expose-Headers").get(0)); @@ -175,7 +176,7 @@ public class TableEndpointUnitTest extends AbstractUnitTest { /* test */ assertThrows(TableNotFoundException.class, () -> { - tableEndpoint.getData(DATABASE_3_ID, TABLE_8_ID, null, null, null, null); + tableEndpoint.getData(DATABASE_3_ID, TABLE_8_ID, null, null, null, httpServletRequest, null); }); } diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/ViewEndpointUnitTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/ViewEndpointUnitTest.java index b9b814378e..543b4a5cf2 100644 --- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/ViewEndpointUnitTest.java +++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/endpoint/ViewEndpointUnitTest.java @@ -161,17 +161,12 @@ public class ViewEndpointUnitTest extends AbstractUnitTest { .thenReturn(DATABASE_1_USER_1_READ_ACCESS_DTO); when(httpServletRequest.getMethod()) .thenReturn("GET"); - when(viewService.count(eq(VIEW_1_PRIVILEGED_DTO), any(Instant.class))) - .thenReturn(VIEW_1_DATA_COUNT); when(viewService.data(eq(VIEW_1_PRIVILEGED_DTO), any(Instant.class), eq(0L), eq(10L))) .thenReturn(VIEW_1_DATA_DTO); /* test */ final ResponseEntity<QueryResultDto> response = viewEndpoint.getData(DATABASE_1_ID, VIEW_1_ID, null, null, null, httpServletRequest, USER_1_PRINCIPAL); assertEquals(HttpStatus.OK, response.getStatusCode()); - assertNotNull(response.getHeaders().get("X-Count")); - assertEquals(1, response.getHeaders().get("X-Count").size()); - assertEquals(VIEW_1_DATA_COUNT, Long.parseLong(response.getHeaders().get("X-Count").get(0))); assertNotNull(response.getHeaders().get("Access-Control-Expose-Headers")); assertEquals(1, response.getHeaders().get("Access-Control-Expose-Headers").size()); assertEquals("X-Count", response.getHeaders().get("Access-Control-Expose-Headers").get(0)); diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/mvc/PrometheusEndpointMvcTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/mvc/PrometheusEndpointMvcTest.java index 9d42c1a54d..5898372633 100644 --- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/mvc/PrometheusEndpointMvcTest.java +++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/mvc/PrometheusEndpointMvcTest.java @@ -126,12 +126,12 @@ public class PrometheusEndpointMvcTest extends AbstractUnitTest { } @Test - @WithMockUser(username = USER_1_USERNAME, authorities = {"dbrepo_subset_list", "execute-query", "persist-query"}) + @WithMockUser(username = USER_1_USERNAME, authorities = {"execute-query", "persist-query"}) public void prometheusSubsetEndpoint_succeeds() { /* mock */ try { - subsetEndpoint.list(DATABASE_1_ID, null, USER_1_PRINCIPAL); + subsetEndpoint.list(DATABASE_1_ID, null); } catch (Exception e) { /* ignore */ } @@ -151,7 +151,7 @@ public class PrometheusEndpointMvcTest extends AbstractUnitTest { /* ignore */ } try { - subsetEndpoint.findById(DATABASE_1_ID, QUERY_1_ID, new MockHttpServletRequest(), null, USER_1_PRINCIPAL); + subsetEndpoint.findById(DATABASE_1_ID, QUERY_1_ID, new MockHttpServletRequest(), null); } catch (Exception e) { /* ignore */ } @@ -171,7 +171,7 @@ public class PrometheusEndpointMvcTest extends AbstractUnitTest { /* mock */ try { - tableEndpoint.getData(DATABASE_1_ID, TABLE_1_ID, null, null, null, null); + tableEndpoint.getData(DATABASE_1_ID, TABLE_1_ID, null, null, null, httpServletRequest, null); } catch (Exception e) { /* ignore */ } diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/SchemaServiceIntegrationTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/SchemaServiceIntegrationTest.java index be1f6b5dae..540e17850a 100644 --- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/SchemaServiceIntegrationTest.java +++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/SchemaServiceIntegrationTest.java @@ -332,12 +332,39 @@ public class SchemaServiceIntegrationTest extends AbstractUnitTest { assertEquals("other_id", pk1.getColumn().getName()); assertEquals("other_id", pk1.getColumn().getInternalName()); assertEquals(ColumnTypeDto.BIGINT, pk1.getColumn().getColumnType()); + } + + @Test + public void inspectTable_exoticBoolean_succeeds() throws TableNotFoundException, SQLException { + /* test */ + final TableDto response = schemaService.inspectTable(DATABASE_1_PRIVILEGED_DTO, "exotic_boolean"); + final ConstraintsDto constraints = response.getConstraints(); + final List<PrimaryKeyDto> primaryKey = new LinkedList<>(constraints.getPrimaryKey()); + assertEquals(1, primaryKey.size()); + final PrimaryKeyDto pk0 = primaryKey.get(0); + assertNull(pk0.getId()); + assertNotNull(pk0.getTable()); + assertNull(pk0.getTable().getId()); + assertEquals("exotic_boolean", pk0.getTable().getName()); + assertEquals("exotic_boolean", pk0.getTable().getInternalName()); + assertNotNull(pk0.getColumn()); + assertNull(pk0.getColumn().getId()); + assertNull(pk0.getColumn().getTableId()); + assertEquals(DATABASE_1_ID, pk0.getColumn().getDatabaseId()); + assertNull(pk0.getColumn().getAlias()); + assertEquals("bool_default", pk0.getColumn().getName()); + assertEquals("bool_default", pk0.getColumn().getInternalName()); + assertEquals(ColumnTypeDto.BOOL, pk0.getColumn().getColumnType()); + final List<ColumnDto> columns = response.getColumns(); + assertEquals(3, columns.size()); + assertColumn(columns.get(0), null, null, DATABASE_1_ID, "bool_default", "bool_default", ColumnTypeDto.BOOL, null, 0L, false, null, null); + assertColumn(columns.get(1), null, null, DATABASE_1_ID, "bool_tinyint", "bool_tinyint", ColumnTypeDto.BOOL, null, 0L, false, null, null); + assertColumn(columns.get(2), null, null, DATABASE_1_ID, "bool_tinyint_unsigned", "bool_tinyint_unsigned", ColumnTypeDto.BOOL, null, 0L, false, null, null); } @Test - public void inspectView_succeeds() throws ViewMalformedException, SQLException, ViewNotFoundException, - ViewSchemaException { + public void inspectView_succeeds() throws SQLException, ViewNotFoundException { /* test */ final ViewDto response = schemaService.inspectView(DATABASE_1_PRIVILEGED_DTO, "not_in_metadata_db2"); diff --git a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/TableServiceIntegrationTest.java b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/TableServiceIntegrationTest.java index 41fa4963fb..b8267e959e 100644 --- a/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/TableServiceIntegrationTest.java +++ b/dbrepo-data-service/rest-service/src/test/java/at/tuwien/service/TableServiceIntegrationTest.java @@ -334,7 +334,7 @@ public class TableServiceIntegrationTest extends AbstractUnitTest { /* test */ final List<TableDto> response = tableService.getSchemas(DATABASE_1_PRIVILEGED_DTO); - assertEquals(3, response.size()); + assertEquals(4, response.size()); final TableDto table0 = response.get(0); Assertions.assertEquals("complex_foreign_keys", table0.getInternalName()); Assertions.assertEquals("complex_foreign_keys", table0.getName()); @@ -411,32 +411,53 @@ public class TableServiceIntegrationTest extends AbstractUnitTest { assertEquals(0, constraints1.getUniques().size()); /* table 2 */ final TableDto table2 = response.get(2); - Assertions.assertEquals("not_in_metadata_db", table2.getInternalName()); - Assertions.assertEquals("not_in_metadata_db", table2.getName()); + Assertions.assertEquals("exotic_boolean", table2.getInternalName()); + Assertions.assertEquals("exotic_boolean", table2.getName()); Assertions.assertEquals(DATABASE_1_ID, table2.getTdbid()); assertTrue(table2.getIsVersioned()); Assertions.assertEquals(DATABASE_1_PUBLIC, table2.getIsPublic()); final List<ColumnDto> columns2 = table2.getColumns(); assertNotNull(columns2); - Assertions.assertEquals(5, columns2.size()); - assertColumn(columns2.get(0), null, null, DATABASE_1_ID, "id", "id", ColumnTypeDto.BIGINT, 19L, 0L, false, null, null); - assertColumn(columns2.get(1), null, null, DATABASE_1_ID, "given_name", "given_name", ColumnTypeDto.VARCHAR, 255L, null, false, null, null); - assertColumn(columns2.get(2), null, null, DATABASE_1_ID, "middle_name", "middle_name", ColumnTypeDto.VARCHAR, 255L, null, true, null, null); - assertColumn(columns2.get(3), null, null, DATABASE_1_ID, "family_name", "family_name", ColumnTypeDto.VARCHAR, 255L, null, false, null, null); - assertColumn(columns2.get(4), null, null, DATABASE_1_ID, "age", "age", ColumnTypeDto.INT, 10L, 0L, false, null, null); + Assertions.assertEquals(3, columns2.size()); + assertColumn(columns2.get(0), null, null, DATABASE_1_ID, "bool_default", "bool_default", ColumnTypeDto.BOOL, null, 0L, false, null, null); + assertColumn(columns2.get(1), null, null, DATABASE_1_ID, "bool_tinyint", "bool_tinyint", ColumnTypeDto.BOOL, null, 0L, false, null, null); + assertColumn(columns2.get(2), null, null, DATABASE_1_ID, "bool_tinyint_unsigned", "bool_tinyint_unsigned", ColumnTypeDto.BOOL, null, 0L, false, null, null); final ConstraintsDto constraints2 = table2.getConstraints(); assertNotNull(constraints2); final Set<PrimaryKeyDto> primaryKey2 = constraints2.getPrimaryKey(); Assertions.assertEquals(1, primaryKey2.size()); final Set<String> checks2 = constraints2.getChecks(); - Assertions.assertEquals(1, checks2.size()); - Assertions.assertEquals(Set.of("`age` > 0 and `age` < 120"), checks2); + Assertions.assertEquals(0, checks2.size()); final List<UniqueDto> uniques2 = constraints2.getUniques(); - Assertions.assertEquals(1, uniques2.size()); - Assertions.assertEquals(2, uniques2.get(0).getColumns().size()); - Assertions.assertEquals("not_in_metadata_db", uniques2.get(0).getTable().getInternalName()); - Assertions.assertEquals("given_name", uniques2.get(0).getColumns().get(0).getInternalName()); - Assertions.assertEquals("family_name", uniques2.get(0).getColumns().get(1).getInternalName()); + Assertions.assertEquals(0, uniques2.size()); + /* table 3 */ + final TableDto table3 = response.get(3); + Assertions.assertEquals("not_in_metadata_db", table3.getInternalName()); + Assertions.assertEquals("not_in_metadata_db", table3.getName()); + Assertions.assertEquals(DATABASE_1_ID, table3.getTdbid()); + assertTrue(table3.getIsVersioned()); + Assertions.assertEquals(DATABASE_1_PUBLIC, table3.getIsPublic()); + final List<ColumnDto> columns3 = table3.getColumns(); + assertNotNull(columns3); + Assertions.assertEquals(5, columns3.size()); + assertColumn(columns3.get(0), null, null, DATABASE_1_ID, "id", "id", ColumnTypeDto.BIGINT, 19L, 0L, false, null, null); + assertColumn(columns3.get(1), null, null, DATABASE_1_ID, "given_name", "given_name", ColumnTypeDto.VARCHAR, 255L, null, false, null, null); + assertColumn(columns3.get(2), null, null, DATABASE_1_ID, "middle_name", "middle_name", ColumnTypeDto.VARCHAR, 255L, null, true, null, null); + assertColumn(columns3.get(3), null, null, DATABASE_1_ID, "family_name", "family_name", ColumnTypeDto.VARCHAR, 255L, null, false, null, null); + assertColumn(columns3.get(4), null, null, DATABASE_1_ID, "age", "age", ColumnTypeDto.INT, 10L, 0L, false, null, null); + final ConstraintsDto constraints3 = table3.getConstraints(); + assertNotNull(constraints3); + final Set<PrimaryKeyDto> primaryKey3 = constraints3.getPrimaryKey(); + Assertions.assertEquals(1, primaryKey3.size()); + final Set<String> checks3 = constraints3.getChecks(); + Assertions.assertEquals(1, checks3.size()); + Assertions.assertEquals(Set.of("`age` > 0 and `age` < 120"), checks3); + final List<UniqueDto> uniques3 = constraints3.getUniques(); + Assertions.assertEquals(1, uniques3.size()); + Assertions.assertEquals(2, uniques3.get(0).getColumns().size()); + Assertions.assertEquals("not_in_metadata_db", uniques3.get(0).getTable().getInternalName()); + Assertions.assertEquals("given_name", uniques3.get(0).getColumns().get(0).getInternalName()); + Assertions.assertEquals("family_name", uniques3.get(0).getColumns().get(1).getInternalName()); } @Test diff --git a/dbrepo-data-service/rest-service/src/test/resources/init/weather.sql b/dbrepo-data-service/rest-service/src/test/resources/init/weather.sql index 79755c97bf..7c3ca99ce3 100644 --- a/dbrepo-data-service/rest-service/src/test/resources/init/weather.sql +++ b/dbrepo-data-service/rest-service/src/test/resources/init/weather.sql @@ -32,7 +32,7 @@ CREATE TABLE complex_foreign_keys ( id BIGINT NOT NULL PRIMARY KEY, weather_id BIGINT NOT NULL, - other_id BIGINT NOT NULL, + other_id BIGINT NOT NULL, FOREIGN KEY (weather_id, other_id) REFERENCES complex_primary_key (id, `other_id`) ) WITH SYSTEM VERSIONING; @@ -42,6 +42,13 @@ CREATE TABLE sensor `value` DECIMAL ) WITH SYSTEM VERSIONING; +CREATE TABLE exotic_boolean +( + `bool_default` BOOLEAN NOT NULL PRIMARY KEY, + `bool_tinyint` TINYINT(1) NOT NULL, + `bool_tinyint_unsigned` TINYINT(1) UNSIGNED NOT NULL +) WITH SYSTEM VERSIONING; + INSERT INTO weather_location (location, lat, lng) VALUES ('Albury', -36.0653583, 146.9112214), ('Sydney', -33.847927, 150.6517942), diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/KeycloakGatewayImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/KeycloakGatewayImpl.java index 6e233395d9..1d73f6219a 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/KeycloakGatewayImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/KeycloakGatewayImpl.java @@ -58,20 +58,20 @@ public class KeycloakGatewayImpl implements KeycloakGateway { .exchange(url, HttpMethod.POST, new HttpEntity<>(payload, headers), TokenDto.class); } catch (HttpServerErrorException e) { log.error("Failed to obtain user token: {}", e.getMessage()); - throw new AuthServiceConnectionException("Service unavailable", e); + throw new AuthServiceConnectionException("Failed to obtain user token: " + e.getMessage(), e); } catch (HttpClientErrorException.BadRequest e) { if (e.getResponseBodyAsByteArray() != null && e.getResponseBodyAsByteArray().length > 0) { final KeycloakErrorDto error = e.getResponseBodyAs(KeycloakErrorDto.class); if (error != null && error.getError().equals("invalid_grant")) { log.error("Failed to obtain user token: {}", error.getErrorDescription()); - throw new AccountNotSetupException(error.getErrorDescription()); + throw new AccountNotSetupException("Failed to obtain user token: " + error.getErrorDescription(), e); } } log.error("Failed to obtain user token: bad request"); throw new CredentialsInvalidException("Bad request", e); } catch (HttpClientErrorException.Unauthorized e) { log.error("Failed to obtain user token: invalid credentials"); - throw new CredentialsInvalidException("Invalid credentials", e); + throw new CredentialsInvalidException("Invalid credentials: " + e.getMessage(), e); } return response.getBody(); } diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/MetadataServiceGatewayImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/MetadataServiceGatewayImpl.java index 06641b738b..b4cb2ff504 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/MetadataServiceGatewayImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/gateway/impl/MetadataServiceGatewayImpl.java @@ -7,7 +7,6 @@ import at.tuwien.api.database.ViewDto; import at.tuwien.api.database.internal.PrivilegedDatabaseDto; import at.tuwien.api.database.internal.PrivilegedViewDto; import at.tuwien.api.database.table.TableDto; -import at.tuwien.api.database.table.TableStatisticDto; import at.tuwien.api.database.table.internal.PrivilegedTableDto; import at.tuwien.api.identifier.IdentifierDto; import at.tuwien.api.user.PrivilegedUserDto; @@ -56,7 +55,7 @@ public class MetadataServiceGatewayImpl implements MetadataServiceGateway { throw new RemoteUnavailableException("Failed to find container: " + e.getMessage(), e); } catch (HttpClientErrorException.NotFound e) { log.error("Failed to find container with id {}: {}", containerId, e.getMessage()); - throw new ContainerNotFoundException("Failed to find container: " + e.getMessage()); + throw new ContainerNotFoundException("Failed to find container: " + e.getMessage(), e); } if (response.getStatusCode() != HttpStatus.OK) { log.error("Failed to find container with id {}: service responded unsuccessful: {}", containerId, response.getStatusCode()); @@ -88,7 +87,7 @@ public class MetadataServiceGatewayImpl implements MetadataServiceGateway { throw new RemoteUnavailableException("Failed to find database: " + e.getMessage(), e); } catch (HttpClientErrorException.NotFound e) { log.error("Failed to find database with id {}: body is null", id); - throw new DatabaseNotFoundException("Failed to find database: body is null", e); + throw new DatabaseNotFoundException("Failed to find database: body is null: " + e.getMessage(), e); } if (response.getStatusCode() != HttpStatus.OK) { log.error("Failed to find database with id {}: service responded unsuccessful: {}", id, response.getStatusCode()); @@ -141,7 +140,7 @@ public class MetadataServiceGatewayImpl implements MetadataServiceGateway { throw new RemoteUnavailableException("Failed to find table: " + e.getMessage(), e); } catch (HttpClientErrorException.NotFound e) { log.error("Failed to find table with id {}: not found: {}", id, e.getMessage()); - throw new TableNotFoundException("Failed to find table: " + e.getMessage()); + throw new TableNotFoundException("Failed to find table: " + e.getMessage(), e); } if (!response.getStatusCode().equals(HttpStatus.OK)) { log.error("Failed to find table with id {}: service responded unsuccessful: {}", id, response.getStatusCode()); @@ -179,7 +178,7 @@ public class MetadataServiceGatewayImpl implements MetadataServiceGateway { throw new RemoteUnavailableException("Failed to find view: " + e.getMessage(), e); } catch (HttpClientErrorException.NotFound e) { log.error("Failed to find view with id {}: not found: {}", id, e.getMessage()); - throw new ViewNotFoundException("Failed to find view: " + e.getMessage()); + throw new ViewNotFoundException("Failed to find view: " + e.getMessage(), e); } if (!response.getStatusCode().equals(HttpStatus.OK)) { log.error("Failed to find view with id {}: service responded unsuccessful: {}", id, response.getStatusCode()); @@ -214,7 +213,7 @@ public class MetadataServiceGatewayImpl implements MetadataServiceGateway { throw new RemoteUnavailableException("Failed to find user: " + e.getMessage(), e); } catch (HttpClientErrorException.NotFound e) { log.error("Failed to find user with id {}: not found: {}", userId, e.getMessage()); - throw new UserNotFoundException("Failed to find user: " + e.getMessage()); + throw new UserNotFoundException("Failed to find user: " + e.getMessage(), e); } if (!response.getStatusCode().equals(HttpStatus.OK)) { log.error("Failed to find user with id {}: service responded unsuccessful: {}", userId, response.getStatusCode()); @@ -238,7 +237,7 @@ public class MetadataServiceGatewayImpl implements MetadataServiceGateway { throw new RemoteUnavailableException("Failed to find user: " + e.getMessage(), e); } catch (HttpClientErrorException.NotFound e) { log.error("Failed to find user with id {}: not found: {}", userId, e.getMessage()); - throw new UserNotFoundException("Failed to find user: " + e.getMessage()); + throw new UserNotFoundException("Failed to find user: " + e.getMessage(), e); } if (!response.getStatusCode().equals(HttpStatus.OK)) { log.error("Failed to find user with id {}: service responded unsuccessful: {}", userId, response.getStatusCode()); diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/DataMapper.java b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/DataMapper.java index 796ef3edeb..b34053ec88 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/DataMapper.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/DataMapper.java @@ -21,6 +21,7 @@ import at.tuwien.api.database.table.constraints.foreign.ForeignKeyReferenceDto; import at.tuwien.api.database.table.constraints.foreign.ReferenceTypeDto; import at.tuwien.api.database.table.constraints.primary.PrimaryKeyDto; import at.tuwien.api.database.table.constraints.unique.UniqueDto; +import at.tuwien.api.user.UserDto; import at.tuwien.config.QueryConfig; import at.tuwien.exception.QueryNotFoundException; import at.tuwien.exception.TableNotFoundException; @@ -97,7 +98,8 @@ public interface DataMapper { /** * Map the inspected schema to either an existing view/table and append e.g. column or (if not existing) create a new view/table. - * @param database The database. + * + * @param database The database. * @param resultSet The inspected schema. * @return The database containing the updated view/table. * @throws SQLException @@ -152,31 +154,23 @@ public interface DataMapper { .databaseId(table.getTdbid()) .description(resultSet.getString(11)) .build(); + final String dataType = resultSet.getString(8); if (column.getColumnType().equals(ColumnTypeDto.ENUM)) { - column.setEnums(Arrays.stream(resultSet.getString(8) - .substring(0, resultSet.getString(8).length() - 1) + column.setEnums(Arrays.stream(dataType.substring(0, resultSet.getString(8).length() - 1) .replace("enum(", "") .split(",")) .map(value -> value.replace("'", "")) .toList()); } if (column.getColumnType().equals(ColumnTypeDto.SET)) { - column.setSets(Arrays.stream(resultSet.getString(8) - .substring(0, resultSet.getString(8).length() - 1) + column.setSets(Arrays.stream(dataType.substring(0, dataType.length() - 1) .replace("set(", "") .split(",")) .map(value -> value.replace("'", "")) .toList()); } - /* constraints */ - if (resultSet.getString(9) != null && resultSet.getString(9).equals("PRI")) { - table.getConstraints().getPrimaryKey().add(PrimaryKeyDto.builder() - .table(tableDtoToTableBriefDto(table)) - .column(columnDtoToColumnBriefDto(column)) - .build()); - } /* fix boolean and set size for others */ - if (resultSet.getString(8).equalsIgnoreCase("tinyint(1)")) { + if (dataType.startsWith("tinyint(1)")) { column.setColumnType(ColumnTypeDto.BOOL); } else if (resultSet.getString(5) != null) { column.setSize(resultSet.getLong(5)); @@ -196,6 +190,13 @@ public interface DataMapper { .id(queryConfig.getDefaultTimeFormatId()) .build()); } + /* constraints */ + if (resultSet.getString(9) != null && resultSet.getString(9).equals("PRI")) { + table.getConstraints().getPrimaryKey().add(PrimaryKeyDto.builder() + .table(tableDtoToTableBriefDto(table)) + .column(columnDtoToColumnBriefDto(column)) + .build()); + } table.getColumns() .add(column); return table; @@ -241,8 +242,9 @@ public interface DataMapper { /** * Parse columns from a SQL statement of a known database. + * * @param database The database. - * @param query The SQL statement. + * @param query The SQL statement. * @return The list of columns. * @throws JSQLParserException The table/view or column was not found in the database. */ @@ -401,6 +403,9 @@ public interface DataMapper { .created(LocalDateTime.parse(data.getString(2), mariaDbFormatter) .atZone(ZoneId.of("UTC")) .toInstant()) + .creator(UserDto.builder() + .id(UUID.fromString(data.getString(3))) + .build()) .createdBy(UUID.fromString(data.getString(3))) .query(data.getString(4)) .queryHash(data.getString(5)) diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java index 74e9ffe66d..a88cdb5078 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/mapper/MariaDbMapper.java @@ -73,6 +73,30 @@ public interface MariaDbMapper { return statement.toString(); } + default String databaseRevokePrivilegesQuery(String username) { + final StringBuilder statement = new StringBuilder("REVOKE ALL PRIVILEGES ON *.* FROM `") + .append(username) + .append("`@`%`;"); + log.trace("mapped revoke privileges statement: {}", statement); + return statement.toString(); + } + + default String databaseGrantProcedureQuery(String username, String procedure) { + final StringBuilder statement = new StringBuilder("GRANT EXECUTE ON PROCEDURE `") + .append(procedure) + .append("` TO `") + .append(username) + .append("`@`%`;"); + log.trace("mapped revoke privileges statement: {}", statement); + return statement.toString(); + } + + default String databaseFlushPrivilegesQuery() { + final String statement = "FLUSH PRIVILEGES;"; + log.trace("mapped flush privileges statement: {}", statement); + return statement; + } + @Named("createDatabase") default String databaseCreateDatabaseQuery(String database) { final StringBuilder statement = new StringBuilder("CREATE DATABASE `") @@ -82,6 +106,60 @@ public interface MariaDbMapper { return statement.toString(); } + default String queryStoreCreateSequenceRawQuery() { + final String statement = "CREATE SEQUENCE `qs_queries_seq` NOCACHE;"; + log.trace("mapped create query store sequence statement: {}", statement); + return statement; + } + + default String queryStoreCreateTableRawQuery() { + final String statement = "CREATE TABLE `qs_queries` ( `id` bigint not null primary key default nextval(`qs_queries_seq`), `created` datetime not null default now(), `executed` datetime not null default now(), `created_by` varchar(36) not null, `query` text not null, `query_normalized` text not null, `is_persisted` boolean not null, `query_hash` varchar(255) not null, `result_hash` varchar(255), `result_number` bigint);"; + log.trace("mapped create query store table statement: {}", statement); + return statement; + } + + default String queryStoreCreateHashTableProcedureRawQuery() { + final String statement = "CREATE PROCEDURE hash_table(IN name VARCHAR(255), OUT hash VARCHAR(255), OUT count BIGINT) BEGIN DECLARE _sql TEXT; SELECT CONCAT('SELECT SHA2(GROUP_CONCAT(CONCAT_WS(\\'\\',', GROUP_CONCAT(CONCAT('`', column_name, '`') ORDER BY column_name), ') SEPARATOR \\',\\'), 256) AS hash, COUNT(*) AS count FROM `', name, '` INTO @hash, @count;') FROM `information_schema`.`columns` WHERE `table_schema` = DATABASE() AND `table_name` = name INTO _sql; PREPARE stmt FROM _sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; SET hash = @hash; SET count = @count; END;"; + log.trace("mapped create query store hash_table procedure statement: {}", statement); + return statement; + } + + default String queryStoreCreateStoreQueryProcedureRawQuery() { + final String statement = "CREATE PROCEDURE store_query(IN query TEXT, IN executed DATETIME, OUT queryId BIGINT) BEGIN DECLARE _queryhash varchar(255) DEFAULT SHA2(query, 256); DECLARE _username varchar(255) DEFAULT REGEXP_REPLACE(current_user(), '@.*', ''); DECLARE _query TEXT DEFAULT CONCAT('CREATE OR REPLACE TABLE _tmp AS (', query, ')'); PREPARE stmt FROM _query; EXECUTE stmt; DEALLOCATE PREPARE stmt; CALL hash_table('_tmp', @hash, @count); DROP TABLE IF EXISTS `_tmp`; IF @hash IS NULL THEN INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); ELSE INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); END IF; END;"; + log.trace("mapped create query store store_query procedure statement: {}", statement); + return statement; + } + + default String queryStoreCreateInternalStoreQueryProcedureRawQuery() { + final String statement = "CREATE DEFINER = 'root' PROCEDURE _store_query(IN _username VARCHAR(255), IN query TEXT, IN executed DATETIME, OUT queryId BIGINT) BEGIN DECLARE _queryhash varchar(255) DEFAULT SHA2(query, 256); DECLARE _query TEXT DEFAULT CONCAT('CREATE OR REPLACE TABLE _tmp AS (', query, ')'); PREPARE stmt FROM _query; EXECUTE stmt; DEALLOCATE PREPARE stmt; CALL hash_table('_tmp', @hash, @count); DROP TABLE IF EXISTS `_tmp`; IF @hash IS NULL THEN INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); ELSE INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); END IF; END;"; + log.trace("mapped create query store _store_query procedure statement: {}", statement); + return statement; + } + + default String queryStoreStoreQueryRawQuery() { + final String statement = "{call _store_query(?, ?, ?, ?)}"; + log.trace("mapped store query statement: {}", statement); + return statement; + } + + default String queryStoreUpdateQueryRawQuery() { + final String statement = "UPDATE `qs_queries` SET `is_persisted` = ? WHERE `id` = ?"; + log.trace("mapped update query statement: {}", statement); + return statement; + } + + default String queryStoreDeleteStaleQueriesRawQuery() { + final String statement = "DELETE FROM `qs_queries` WHERE `is_persisted` = false AND ABS(DATEDIFF(`created`, NOW())) >= 1"; + log.trace("mapped delete stale queries statement: {}", statement); + return statement; + } + + default String queryStoreFindQueryRawQuery() { + final String statement = "SELECT `id`, `created`, `created_by`, `query`, `query_hash`, `result_hash`, `result_number`, `is_persisted`, `executed` FROM `qs_queries` q WHERE q.`id` = ?"; + log.trace("mapped find query statement: {}", statement); + return statement; + } + default String databaseTablesSelectRawQuery() { final String statement = "SELECT DISTINCT t.`TABLE_NAME` FROM information_schema.TABLES t WHERE t.`TABLE_SCHEMA` = ? AND t.`TABLE_TYPE` = 'SYSTEM VERSIONED' AND t.`TABLE_NAME` != 'qs_queries' ORDER BY t.`TABLE_NAME` ASC"; log.trace("mapped select tables statement: {}", statement); diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/ViewService.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/ViewService.java index 5acca0018d..f4bef4f067 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/ViewService.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/ViewService.java @@ -20,12 +20,10 @@ public interface ViewService { * @return The list of view metadata. * @throws SQLException * @throws DatabaseMalformedException - * @throws ViewMalformedException * @throws ViewNotFoundException - * @throws ViewSchemaException */ List<ViewDto> getSchemas(PrivilegedDatabaseDto database) throws SQLException, DatabaseMalformedException, - ViewMalformedException, ViewNotFoundException, ViewSchemaException; + ViewNotFoundException; /** * Creates a view in the given data database. diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/AccessServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/AccessServiceMariaDbImpl.java index 8c52e02010..4fdb8c32a6 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/AccessServiceMariaDbImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/AccessServiceMariaDbImpl.java @@ -40,17 +40,25 @@ public class AccessServiceMariaDbImpl extends HibernateConnector implements Acce final Connection connection = dataSource.getConnection(); try { /* create user if not exists */ + long start = System.currentTimeMillis(); connection.prepareStatement(mariaDbMapper.databaseCreateUserQuery(user.getUsername(), user.getPassword())) .execute(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); /* grant access */ final String grants = access != AccessTypeDto.READ ? grantDefaultWrite : grantDefaultRead; + start = System.currentTimeMillis(); connection.prepareStatement(mariaDbMapper.databaseGrantPrivilegesQuery(user.getUsername(), grants)) .execute(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); /* grant query store */ - connection.prepareStatement("GRANT EXECUTE ON PROCEDURE `store_query` TO `" + user.getUsername() + "`@`%`;") + start = System.currentTimeMillis(); + connection.prepareStatement(mariaDbMapper.databaseGrantProcedureQuery(user.getUsername(), "store_query")) .execute(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); /* apply access rights */ - connection.prepareStatement("FLUSH PRIVILEGES;"); + start = System.currentTimeMillis(); + connection.prepareStatement(mariaDbMapper.databaseFlushPrivilegesQuery()); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); } catch (SQLException e) { connection.rollback(); @@ -71,10 +79,12 @@ public class AccessServiceMariaDbImpl extends HibernateConnector implements Acce try { /* grant access */ final String grants = access != AccessTypeDto.READ ? grantDefaultWrite : grantDefaultRead; + final long start = System.currentTimeMillis(); connection.prepareStatement(mariaDbMapper.databaseGrantPrivilegesQuery(user.getUsername(), grants)) .execute(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); /* apply access rights */ - connection.prepareStatement("FLUSH PRIVILEGES;"); + connection.prepareStatement(mariaDbMapper.databaseFlushPrivilegesQuery()); connection.commit(); } catch (SQLException e) { connection.rollback(); @@ -93,10 +103,15 @@ public class AccessServiceMariaDbImpl extends HibernateConnector implements Acce final Connection connection = dataSource.getConnection(); try { /* revoke access */ - connection.prepareStatement("REVOKE ALL PRIVILEGES ON *.* FROM `" + user.getUsername() + "`@`%`;") + long start = System.currentTimeMillis(); + connection.prepareStatement(mariaDbMapper.databaseRevokePrivilegesQuery(user.getUsername())) .execute(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); /* apply access rights */ - connection.prepareStatement("FLUSH PRIVILEGES;"); + start = System.currentTimeMillis(); + connection.prepareStatement(mariaDbMapper.databaseFlushPrivilegesQuery()) + .execute(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); } catch (SQLException e) { connection.rollback(); diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DatabaseServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DatabaseServiceMariaDbImpl.java index 4ee483ad5e..2d58744a21 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DatabaseServiceMariaDbImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/DatabaseServiceMariaDbImpl.java @@ -43,8 +43,10 @@ public class DatabaseServiceMariaDbImpl extends HibernateConnector implements Da final Connection connection = dataSource.getConnection(); try { /* create database if not exists */ + final long start = System.currentTimeMillis(); connection.prepareStatement(mariaDbMapper.databaseCreateDatabaseQuery(data.getInternalName())) .execute(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); } catch (SQLException e) { connection.rollback(); @@ -77,8 +79,10 @@ public class DatabaseServiceMariaDbImpl extends HibernateConnector implements Da final Connection connection = dataSource.getConnection(); try { /* update user password */ + final long start = System.currentTimeMillis(); connection.prepareStatement(mariaDbMapper.databaseSetPasswordQuery(data.getUsername(), data.getPassword())) .execute(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); } catch (SQLException e) { connection.rollback(); diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/HibernateConnector.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/HibernateConnector.java index 6a583b274a..22bb599c60 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/HibernateConnector.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/HibernateConnector.java @@ -12,6 +12,7 @@ import org.springframework.stereotype.Service; public abstract class HibernateConnector { public static ComboPooledDataSource getPrivilegedDataSource(PrivilegedContainerDto container, String databaseName) { + final long start = System.currentTimeMillis(); final ComboPooledDataSource dataSource = new ComboPooledDataSource(); dataSource.setJdbcUrl(url(container, databaseName)); dataSource.setUser(container.getUsername()); @@ -21,7 +22,7 @@ public abstract class HibernateConnector { dataSource.setAcquireIncrement(5); dataSource.setMaxPoolSize(20); dataSource.setMaxStatements(100); - log.trace("created pooled data source {} (user={}, password=(hidden))", url(container, databaseName), container.getUsername()); + log.trace("created pooled data source {} in {} ms (user={}, password=(hidden))", url(container, databaseName), System.currentTimeMillis() - start, container.getUsername()); return dataSource; } diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/QueueServiceRabbitMqImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/QueueServiceRabbitMqImpl.java index f2675d4e5b..0b1dc7caa1 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/QueueServiceRabbitMqImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/QueueServiceRabbitMqImpl.java @@ -46,7 +46,9 @@ public class QueueServiceRabbitMqImpl extends HibernateConnector implements Queu dataMapper.prepareStatementWithColumnTypeObject(preparedStatement, optional.get().getColumnType(), idx[0]++, entry.getValue()); } + final long start = System.currentTimeMillis(); preparedStatement.executeUpdate(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); log.trace("successfully inserted tuple"); } finally { dataSource.close(); diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SchemaServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SchemaServiceMariaDbImpl.java index cc5840080b..a0fbd1434c 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SchemaServiceMariaDbImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SchemaServiceMariaDbImpl.java @@ -49,26 +49,32 @@ public class SchemaServiceMariaDbImpl extends HibernateConnector implements Sche final Connection connection = dataSource.getConnection(); try { /* obtain only table metadata */ + long start = System.currentTimeMillis(); final PreparedStatement statement1 = connection.prepareStatement(mariaDbMapper.databaseTableSelectRawQuery()); statement1.setString(1, database.getInternalName()); statement1.setString(2, tableName); log.trace("1={}, 2={}", database.getInternalName(), tableName); TableDto table = dataMapper.schemaResultSetToTable(metadataMapper.privilegedDatabaseDtoToDatabaseDto(database), statement1.executeQuery()); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); /* obtain columns metadata */ + start = System.currentTimeMillis(); final PreparedStatement statement2 = connection.prepareStatement(mariaDbMapper.databaseTableColumnsSelectRawQuery()); statement2.setString(1, database.getInternalName()); statement2.setString(2, tableName); log.trace("1={}, 2={}", database.getInternalName(), tableName); final ResultSet resultSet2 = statement2.executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); while (resultSet2.next()) { table = dataMapper.resultSetToTable(resultSet2, table, queryConfig); } /* obtain check constraints metadata */ + start = System.currentTimeMillis(); final PreparedStatement statement3 = connection.prepareStatement(mariaDbMapper.columnsCheckConstraintSelectRawQuery()); statement3.setString(1, database.getInternalName()); statement3.setString(2, tableName); log.trace("1={}, 2={}", database.getInternalName(), tableName); final ResultSet resultSet3 = statement3.executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); while (resultSet3.next()) { final String clause = resultSet3.getString(1); table.getConstraints() @@ -77,11 +83,13 @@ public class SchemaServiceMariaDbImpl extends HibernateConnector implements Sche log.trace("found check clause: {}", clause); } /* obtain column constraints metadata */ + start = System.currentTimeMillis(); final PreparedStatement statement4 = connection.prepareStatement(mariaDbMapper.databaseTableConstraintsSelectRawQuery()); statement4.setString(1, database.getInternalName()); statement4.setString(2, tableName); log.trace("1={}, 2={}", database.getInternalName(), tableName); final ResultSet resultSet4 = statement4.executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); while (resultSet4.next()) { table = dataMapper.resultSetToConstraint(resultSet4, table); for (UniqueDto uk : table.getConstraints().getUniques()) { @@ -122,11 +130,13 @@ public class SchemaServiceMariaDbImpl extends HibernateConnector implements Sche final DatabaseDto database = metadataMapper.privilegedDatabaseDtoToDatabaseDto(privilegedDatabase); try { /* obtain only view metadata */ + long start = System.currentTimeMillis(); final PreparedStatement statement1 = connection.prepareStatement(mariaDbMapper.databaseViewSelectRawQuery()); statement1.setString(1, database.getInternalName()); statement1.setString(2, viewName); log.trace("1={}, 2={}", database.getInternalName(), viewName); final ResultSet resultSet1 = statement1.executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); if (!resultSet1.next()) { throw new ViewNotFoundException("Failed to find view in the information schema"); } @@ -136,11 +146,13 @@ public class SchemaServiceMariaDbImpl extends HibernateConnector implements Sche view.setCreator(database.getCreator()); view.setCreatedBy(database.getCreator().getId()); /* obtain view columns */ + start = System.currentTimeMillis(); final PreparedStatement statement2 = connection.prepareStatement(mariaDbMapper.databaseTableColumnsSelectRawQuery()); statement2.setString(1, database.getInternalName()); statement2.setString(2, viewName); log.trace("1={}, 2={}", database.getInternalName(), viewName); final ResultSet resultSet2 = statement2.executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); TableDto tmp = TableDto.builder() .columns(new LinkedList<>()) .build(); diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java index 3d19276196..53d93d35df 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/SubsetServiceMariaDbImpl.java @@ -68,16 +68,26 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs final Connection connection = dataSource.getConnection(); try { /* create query store */ - connection.prepareStatement("CREATE SEQUENCE `qs_queries_seq` NOCACHE;") + long start = System.currentTimeMillis(); + connection.prepareStatement(mariaDbMapper.queryStoreCreateSequenceRawQuery()) .execute(); - connection.prepareStatement("CREATE TABLE `qs_queries` ( `id` bigint not null primary key default nextval(`qs_queries_seq`), `created` datetime not null default now(), `executed` datetime not null default now(), `created_by` varchar(36) not null, `query` text not null, `query_normalized` text not null, `is_persisted` boolean not null, `query_hash` varchar(255) not null, `result_hash` varchar(255), `result_number` bigint );") + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); + start = System.currentTimeMillis(); + connection.prepareStatement(mariaDbMapper.queryStoreCreateTableRawQuery()) .execute(); - connection.prepareStatement("CREATE PROCEDURE hash_table(IN name VARCHAR(255), OUT hash VARCHAR(255), OUT count BIGINT) BEGIN DECLARE _sql TEXT; SELECT CONCAT('SELECT SHA2(GROUP_CONCAT(CONCAT_WS(\\'\\',', GROUP_CONCAT(CONCAT('`', column_name, '`') ORDER BY column_name), ') SEPARATOR \\',\\'), 256) AS hash, COUNT(*) AS count FROM `', name, '` INTO @hash, @count;') FROM `information_schema`.`columns` WHERE `table_schema` = DATABASE() AND `table_name` = name INTO _sql; PREPARE stmt FROM _sql; EXECUTE stmt; DEALLOCATE PREPARE stmt; SET hash = @hash; SET count = @count; END;") + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); + start = System.currentTimeMillis(); + connection.prepareStatement(mariaDbMapper.queryStoreCreateHashTableProcedureRawQuery()) .execute(); - connection.prepareStatement("CREATE PROCEDURE store_query(IN query TEXT, IN executed DATETIME, OUT queryId BIGINT) BEGIN DECLARE _queryhash varchar(255) DEFAULT SHA2(query, 256); DECLARE _username varchar(255) DEFAULT REGEXP_REPLACE(current_user(), '@.*', ''); DECLARE _query TEXT DEFAULT CONCAT('CREATE OR REPLACE TABLE _tmp AS (', query, ')'); PREPARE stmt FROM _query; EXECUTE stmt; DEALLOCATE PREPARE stmt; CALL hash_table('_tmp', @hash, @count); DROP TABLE IF EXISTS `_tmp`; IF @hash IS NULL THEN INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); ELSE INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); END IF; END;") + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); + start = System.currentTimeMillis(); + connection.prepareStatement(mariaDbMapper.queryStoreCreateStoreQueryProcedureRawQuery()) .execute(); - connection.prepareStatement("CREATE DEFINER = 'root' PROCEDURE _store_query(IN _username VARCHAR(255), IN query TEXT, IN executed DATETIME, OUT queryId BIGINT) BEGIN DECLARE _queryhash varchar(255) DEFAULT SHA2(query, 256); DECLARE _query TEXT DEFAULT CONCAT('CREATE OR REPLACE TABLE _tmp AS (', query, ')'); PREPARE stmt FROM _query; EXECUTE stmt; DEALLOCATE PREPARE stmt; CALL hash_table('_tmp', @hash, @count); DROP TABLE IF EXISTS `_tmp`; IF @hash IS NULL THEN INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` IS NULL); ELSE INSERT INTO `qs_queries` (`created_by`, `query`, `query_normalized`, `is_persisted`, `query_hash`, `result_hash`, `result_number`, `executed`) SELECT _username, query, query, false, _queryhash, @hash, @count, executed WHERE NOT EXISTS (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); SET queryId = (SELECT `id` FROM `qs_queries` WHERE `query_hash` = _queryhash AND `result_hash` = @hash); END IF; END;") + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); + start = System.currentTimeMillis(); + connection.prepareStatement(mariaDbMapper.queryStoreCreateInternalStoreQueryProcedureRawQuery()) .execute(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); } catch (SQLException e) { connection.rollback(); @@ -130,12 +140,14 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs final ComboPooledDataSource dataSource = getPrivilegedDataSource(database); final Connection connection = dataSource.getConnection(); try { + final long start = System.currentTimeMillis(); final PreparedStatement statement = connection.prepareStatement(mariaDbMapper.filterToGetQueriesRawQuery(filterPersisted)); if (filterPersisted != null) { statement.setBoolean(1, filterPersisted); log.trace("filter persisted only {}", filterPersisted); } final ResultSet resultSet = statement.executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); final List<QueryDto> queries = new LinkedList<>(); while (resultSet.next()) { final QueryDto query = dataMapper.resultSetToQueryDto(resultSet); @@ -168,12 +180,18 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs final Connection connection = dataSource.getConnection(); try { /* export to data database sidecar */ + long start = System.currentTimeMillis(); connection.prepareStatement(mariaDbMapper.subsetToRawTemporaryViewQuery(viewName, query.getQuery())) .executeUpdate(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); + start = System.currentTimeMillis(); connection.prepareStatement(mariaDbMapper.subsetToRawExportQuery(viewName, timestamp, filePath)) .executeUpdate(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); + start = System.currentTimeMillis(); connection.prepareStatement(mariaDbMapper.dropViewRawQuery(viewName)) .executeUpdate(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); } catch (SQLException e) { connection.rollback(); @@ -191,8 +209,10 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs final ComboPooledDataSource dataSource = getPrivilegedDataSource(database); final Connection connection = dataSource.getConnection(); try { + final long start = System.currentTimeMillis(); final PreparedStatement preparedStatement = connection.prepareStatement(statement); final ResultSet resultSet = preparedStatement.executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); return dataMapper.resultListToQueryResultDto(columns, resultSet); } catch (SQLException e) { log.error("Failed to execute and map time-versioned query: {}", e.getMessage()); @@ -208,8 +228,10 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs final ComboPooledDataSource dataSource = getPrivilegedDataSource(database); final Connection connection = dataSource.getConnection(); try { + final long start = System.currentTimeMillis(); final ResultSet resultSet = connection.prepareStatement(mariaDbMapper.countRawSelectQuery(statement, timestamp)) .executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); return mariaDbMapper.resultSetToNumber(resultSet); } catch (SQLException e) { log.error("Failed to map object: {}", e.getMessage()); @@ -225,9 +247,11 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs final ComboPooledDataSource dataSource = getPrivilegedDataSource(database); final Connection connection = dataSource.getConnection(); try { - final PreparedStatement preparedStatement = connection.prepareStatement("SELECT `id`, `created`, `created_by`, `query`, `query_hash`, `result_hash`, `result_number`, `is_persisted`, `executed` FROM `qs_queries` q WHERE q.`id` = ?"); + final long start = System.currentTimeMillis(); + final PreparedStatement preparedStatement = connection.prepareStatement(mariaDbMapper.queryStoreFindQueryRawQuery()); preparedStatement.setLong(1, queryId); final ResultSet resultSet = preparedStatement.executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); if (!resultSet.next()) { throw new QueryNotFoundException("Failed to find query"); } @@ -255,12 +279,14 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs final Connection connection = dataSource.getConnection(); try { /* insert query into query store */ - final CallableStatement callableStatement = connection.prepareCall("{call _store_query(?, ?, ?, ?)}"); + final long start = System.currentTimeMillis(); + final CallableStatement callableStatement = connection.prepareCall(mariaDbMapper.queryStoreStoreQueryRawQuery()); callableStatement.setString(1, String.valueOf(userId)); callableStatement.setString(2, query); callableStatement.setTimestamp(3, Timestamp.from(timestamp)); callableStatement.registerOutParameter(4, Types.BIGINT); callableStatement.executeUpdate(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); queryId = callableStatement.getLong(4); callableStatement.close(); log.info("Stored query with id {} in database with name {}", queryId, database.getInternalName()); @@ -282,10 +308,12 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs final Connection connection = dataSource.getConnection(); try { /* update query */ - final PreparedStatement preparedStatement = connection.prepareStatement("UPDATE `qs_queries` SET `is_persisted` = ? WHERE `id` = ?"); + final long start = System.currentTimeMillis(); + final PreparedStatement preparedStatement = connection.prepareStatement(mariaDbMapper.queryStoreUpdateQueryRawQuery()); preparedStatement.setBoolean(1, persist); preparedStatement.setLong(2, queryId); preparedStatement.executeUpdate(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); } catch (SQLException e) { log.error("Failed to (un-)persist query: {}", e.getMessage()); throw new QueryStorePersistException("Failed to (un-)persist query", e); @@ -300,8 +328,10 @@ public class SubsetServiceMariaDbImpl extends HibernateConnector implements Subs final ComboPooledDataSource dataSource = getPrivilegedDataSource(database); final Connection connection = dataSource.getConnection(); try { - connection.prepareStatement("DELETE FROM `qs_queries` WHERE `is_persisted` = false AND ABS(DATEDIFF(`created`, NOW())) >= 1") + final long start = System.currentTimeMillis(); + connection.prepareStatement(mariaDbMapper.queryStoreDeleteStaleQueriesRawQuery()) .executeUpdate(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); } catch (SQLException e) { log.error("Failed to delete stale queries: {}", e.getMessage()); throw new QueryStoreGCException("Failed to delete stale queries: " + e.getMessage(), e); diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java index 4dacc1e094..97778e9e11 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/TableServiceMariaDbImpl.java @@ -25,6 +25,7 @@ import org.apache.commons.lang3.RandomStringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import java.io.File; import java.sql.*; import java.time.Instant; import java.util.*; @@ -60,9 +61,11 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table final List<TableDto> tables = new LinkedList<>(); try { /* inspect tables before views */ + final long start = System.currentTimeMillis(); final PreparedStatement statement = connection.prepareStatement(mariaDbMapper.databaseTablesSelectRawQuery()); statement.setString(1, database.getInternalName()); final ResultSet resultSet1 = statement.executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); while (resultSet1.next()) { final String tableName = resultSet1.getString(1); if (database.getTables().stream().anyMatch(t -> t.getInternalName().equals(tableName))) { @@ -92,8 +95,10 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table final TableStatisticDto statistic; try { /* obtain statistic */ + final long start = System.currentTimeMillis(); final ResultSet resultSet = connection.prepareStatement(mariaDbMapper.tableColumnStatisticsSelectRawQuery(table.getColumns(), table.getInternalName())) .executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); statistic = dataMapper.resultSetToTableStatistic(resultSet); final TableDto tmpTable = schemaService.inspectTable(table.getDatabase(), table.getInternalName()); statistic.setAvgRowLength(tmpTable.getAvgRowLength()); @@ -129,8 +134,10 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table final Connection connection = dataSource.getConnection(); try { /* create table if not exists */ + final long start = System.currentTimeMillis(); connection.prepareStatement(mariaDbMapper.tableCreateDtoToCreateTableRawQuery(data)) .execute(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); } catch (SQLException e) { connection.rollback(); @@ -156,8 +163,10 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table final Connection connection = dataSource.getConnection(); try { /* create table if not exists */ + final long start = System.currentTimeMillis(); connection.prepareStatement(mariaDbMapper.dropTableRawQuery(tableName)) .execute(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); } catch (SQLException e) { connection.rollback(); @@ -177,12 +186,16 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table final QueryResultDto queryResult; try { /* find table data */ + long start = System.currentTimeMillis(); final ResultSet resultSet = connection.prepareStatement(mariaDbMapper.selectDatasetRawQuery( table.getDatabase().getInternalName(), table.getInternalName(), table.getColumns(), timestamp, size, page)) .executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); + start = System.currentTimeMillis(); connection.commit(); queryResult = dataMapper.resultListToQueryResultDto(table.getColumns(), resultSet); + log.debug("mapped result in {} ms", System.currentTimeMillis() - start); } catch (SQLException e) { connection.rollback(); log.error("Failed to find data from table {}.{}: {}", table.getDatabase().getInternalName(), table.getInternalName(), e.getMessage()); @@ -203,9 +216,11 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table final List<TableHistoryDto> history; try { /* find table data */ + final long start = System.currentTimeMillis(); final ResultSet resultSet = connection.prepareStatement(mariaDbMapper.selectHistoryRawQuery( table.getDatabase().getInternalName(), table.getInternalName(), size)) .executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); history = dataMapper.resultSetToTableHistory(resultSet); connection.commit(); } catch (SQLException e) { @@ -227,9 +242,11 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table final Long queryResult; try { /* find table data */ + final long start = System.currentTimeMillis(); final ResultSet resultSet = connection.prepareStatement(mariaDbMapper.selectCountRawQuery( table.getDatabase().getInternalName(), table.getInternalName(), timestamp)) .executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); queryResult = mariaDbMapper.resultSetToNumber(resultSet); connection.commit(); } catch (SQLException e) { @@ -253,9 +270,11 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table final Connection connection = dataSource.getConnection(); try { /* import tuple */ - data.setLocation(s3Config.getS3FilePath() + "/" + data.getLocation()); + data.setLocation(s3Config.getS3FilePath() + File.separator + data.getLocation()); + final long start = System.currentTimeMillis(); connection.prepareStatement(mariaDbMapper.datasetToRawInsertQuery(table.getDatabase().getInternalName(), table, data)) .execute(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); } catch (SQLException e) { connection.rollback(); @@ -283,7 +302,9 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table getColumnType(table.getColumns(), column), idx[0], column, data.getKeys().get(column)); idx[0]++; } + final long start = System.currentTimeMillis(); statement.executeUpdate(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); } catch (SQLException e) { connection.rollback(); @@ -325,7 +346,9 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table getColumnType(table.getColumns(), entry.getKey()), idx[0], entry.getKey(), entry.getValue()); idx[0]++; } + final long start = System.currentTimeMillis(); statement.executeUpdate(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); } catch (SQLException e) { connection.rollback(); @@ -359,7 +382,9 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table getColumnType(table.getColumns(), entry.getKey()), idx[0], entry.getKey(), entry.getValue()); idx[0]++; } + final long start = System.currentTimeMillis(); statement.executeUpdate(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); } catch (SQLException e) { connection.rollback(); @@ -392,9 +417,11 @@ public class TableServiceMariaDbImpl extends HibernateConnector implements Table final Connection connection = dataSource.getConnection(); try { /* export to data database sidecar */ + final long start = System.currentTimeMillis(); connection.prepareStatement(mariaDbMapper.tableOrViewToRawExportQuery(table.getDatabase().getInternalName(), table.getInternalName(), table.getColumns(), timestamp, filePath)) .executeUpdate(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); } catch (SQLException e) { connection.rollback(); diff --git a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/ViewServiceMariaDbImpl.java b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/ViewServiceMariaDbImpl.java index 3cdba35f08..06cf42ae6e 100644 --- a/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/ViewServiceMariaDbImpl.java +++ b/dbrepo-data-service/services/src/main/java/at/tuwien/service/impl/ViewServiceMariaDbImpl.java @@ -24,6 +24,7 @@ import org.apache.commons.lang3.RandomStringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import java.io.File; import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.PreparedStatement; @@ -63,7 +64,7 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe @Override public List<ViewDto> getSchemas(PrivilegedDatabaseDto database) throws SQLException, DatabaseMalformedException, - ViewMalformedException, ViewNotFoundException, ViewSchemaException { + ViewNotFoundException { final ComboPooledDataSource dataSource = getPrivilegedDataSource(database); final Connection connection = dataSource.getConnection(); final List<ViewDto> views = new LinkedList<>(); @@ -71,7 +72,9 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe /* inspect tables before views */ final PreparedStatement statement = connection.prepareStatement(mariaDbMapper.databaseViewsSelectRawQuery()); statement.setString(1, database.getInternalName()); + final long start = System.currentTimeMillis(); final ResultSet resultSet1 = statement.executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); while (resultSet1.next()) { final String viewName = resultSet1.getString(1); if (database.getViews().stream().anyMatch(v -> v.getInternalName().equals(viewName))) { @@ -117,8 +120,10 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe .build(); try { /* create view if not exists */ + final long start = System.currentTimeMillis(); connection.prepareStatement(mariaDbMapper.viewCreateRawQuery(view.getInternalName(), data.getQuery())) .execute(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); /* select view columns */ final PreparedStatement statement2 = connection.prepareStatement(mariaDbMapper.databaseTableColumnsSelectRawQuery()); statement2.setString(1, database.getInternalName()); @@ -151,10 +156,12 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe .stream() .map(metadataMapper::viewColumnDtoToColumnDto) .toList(); + final long start = System.currentTimeMillis(); final ResultSet resultSet = connection.prepareStatement( mariaDbMapper.selectDatasetRawQuery(view.getDatabase().getInternalName(), view.getInternalName(), mappedColumns, timestamp, size, page)) .executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); queryResult = dataMapper.resultListToQueryResultDto(mappedColumns, resultSet); queryResult.setId(view.getId()); connection.commit(); @@ -174,8 +181,10 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe final Connection connection = dataSource.getConnection(); try { /* drop view if exists */ + final long start = System.currentTimeMillis(); connection.prepareStatement(mariaDbMapper.dropViewRawQuery(view.getInternalName())) .execute(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); } catch (SQLException e) { connection.rollback(); @@ -196,9 +205,11 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe final Long queryResult; try { /* find view data */ + final long start = System.currentTimeMillis(); final ResultSet resultSet = connection.prepareStatement(mariaDbMapper.selectCountRawQuery( view.getDatabase().getInternalName(), view.getInternalName(), timestamp)) .executeQuery(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); queryResult = mariaDbMapper.resultSetToNumber(resultSet); connection.commit(); } catch (SQLException e) { @@ -217,7 +228,7 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe throws SQLException, QueryMalformedException, StorageNotFoundException, StorageUnavailableException, RemoteUnavailableException, SidecarExportException { final String fileName = RandomStringUtils.randomAlphabetic(40) + ".csv"; - final String filePath = s3Config.getS3FilePath() + "/" + fileName; + final String filePath = s3Config.getS3FilePath() + File.separator + fileName; final ComboPooledDataSource dataSource = getPrivilegedDataSource(database); final Connection connection = dataSource.getConnection(); try { @@ -226,9 +237,11 @@ public class ViewServiceMariaDbImpl extends HibernateConnector implements ViewSe .stream() .map(metadataMapper::viewColumnDtoToColumnDto) .toList(); + final long start = System.currentTimeMillis(); connection.prepareStatement(mariaDbMapper.tableOrViewToRawExportQuery(database.getInternalName(), view.getInternalName(), columns, timestamp, filePath)) .executeUpdate(); + log.debug("executed statement in {} ms", System.currentTimeMillis() - start); connection.commit(); } catch (SQLException e) { connection.rollback(); diff --git a/dbrepo-gateway-service/dbrepo.conf b/dbrepo-gateway-service/dbrepo.conf index f9c0001ceb..de8eaa417b 100644 --- a/dbrepo-gateway-service/dbrepo.conf +++ b/dbrepo-gateway-service/dbrepo.conf @@ -134,7 +134,7 @@ server { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_pass http://data; - proxy_read_timeout 90; + proxy_read_timeout 600; } location ~ /api/(database|concept|container|identifier|image|message|license|oai|ontology|unit|user) { diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/impl/BrokerServiceGatewayImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/impl/BrokerServiceGatewayImpl.java index 307c166feb..da1c289185 100644 --- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/impl/BrokerServiceGatewayImpl.java +++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/impl/BrokerServiceGatewayImpl.java @@ -38,9 +38,9 @@ public class BrokerServiceGatewayImpl implements BrokerServiceGateway { response = restTemplate.exchange(path, HttpMethod.PUT, new HttpEntity<>(data), Void.class); } catch (HttpServerErrorException e) { log.error("Failed to grant topic permissions: {}", e.getMessage()); - throw new BrokerServiceConnectionException("Failed to grant topic permissions: " + e.getMessage()); + throw new BrokerServiceConnectionException("Failed to grant topic permissions: " + e.getMessage(), e); } catch (Exception e) { - log.error("Failed to grant topic permissions: unexpected response: {}", e.getMessage()); + log.error("Failed to grant topic permissions: unexpected response: {}", e.getMessage(), e); throw new BrokerServiceException("Failed to grant topic permissions: unexpected response: " + e.getMessage(), e); } if (!response.getStatusCode().equals(HttpStatus.CREATED) && !response.getStatusCode().equals(HttpStatus.NO_CONTENT)) { @@ -59,7 +59,7 @@ public class BrokerServiceGatewayImpl implements BrokerServiceGateway { response = restTemplate.exchange(path, HttpMethod.PUT, new HttpEntity<>(data), Void.class); } catch (HttpServerErrorException e) { log.error("Failed to grant virtual host permissions: {}", e.getMessage()); - throw new BrokerServiceConnectionException("Failed to grant virtual host permissions: " + e.getMessage()); + throw new BrokerServiceConnectionException("Failed to grant virtual host permissions: " + e.getMessage(), e); } catch (Exception e) { log.error("Failed to grant virtual host permissions: unexpected response: {}", e.getMessage()); throw new BrokerServiceException("Failed to grant virtual host permissions: unexpected response: " + e.getMessage(), e); @@ -80,7 +80,7 @@ public class BrokerServiceGatewayImpl implements BrokerServiceGateway { response = restTemplate.exchange(path, HttpMethod.PUT, new HttpEntity<>(data), Void.class); } catch (HttpServerErrorException e) { log.error("Failed to grant exchange permissions: {}", e.getMessage()); - throw new BrokerServiceConnectionException("Failed to grant exchange permissions: " + e.getMessage()); + throw new BrokerServiceConnectionException("Failed to grant exchange permissions: " + e.getMessage(), e); } catch (Exception e) { log.error("Failed to grant exchange permissions: unexpected response: {}", e.getMessage()); throw new BrokerServiceException("Failed to grant exchange permissions: unexpected response: " + e.getMessage(), e); diff --git a/dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/impl/CrossrefGatewayImpl.java b/dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/impl/CrossrefGatewayImpl.java index 542d9c981d..8fd957f330 100644 --- a/dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/impl/CrossrefGatewayImpl.java +++ b/dbrepo-metadata-service/services/src/main/java/at/tuwien/gateway/impl/CrossrefGatewayImpl.java @@ -37,7 +37,7 @@ public class CrossrefGatewayImpl implements CrossrefGateway { response = restTemplate.exchange(gatewayConfig.getCrossRefEndpoint() + path, HttpMethod.GET, HttpEntity.EMPTY, CrossrefDto.class); } catch (HttpServerErrorException e) { log.error("Failed to retrieve crossref metadata: {}", e.getMessage()); - throw new DoiNotFoundException("Failed to retrieve crossref metadata: " + e.getMessage()); + throw new DoiNotFoundException("Failed to retrieve crossref metadata: " + e.getMessage(), e); } return response.getBody(); } diff --git a/dbrepo-ui/components/database/DatabaseCard.vue b/dbrepo-ui/components/database/DatabaseCard.vue index 9485fdee10..48aefa7493 100644 --- a/dbrepo-ui/components/database/DatabaseCard.vue +++ b/dbrepo-ui/components/database/DatabaseCard.vue @@ -4,11 +4,20 @@ :to="`/database/${database.id}/info`" variant="flat" rounded="0" - :href="`/database/${database.id}`"> + :href="`/database/${database.id}`" + @click="loading = true"> <v-divider class="mx-4" /> - <v-card-title - class="text-primary text-decoration-underline" - v-text="formatTitle(database)" /> + <v-card-title> + <span + class="text-primary text-decoration-underline" + v-text="formatTitle(database)" /> + <v-progress-circular + v-if="loading" + color="primary" + size="24" + class="ml-1" + indeterminate /> + </v-card-title> <v-card-subtitle v-text="formatCreators(database)" /> <v-card-text> @@ -68,6 +77,11 @@ import { formatLanguage } from '@/utils' export default { + data() { + return { + loading: false + } + }, props: { database: { default: () => { diff --git a/dbrepo-ui/components/database/DatabaseToolbar.vue b/dbrepo-ui/components/database/DatabaseToolbar.vue index 5a86bbaa01..0c87e5599c 100644 --- a/dbrepo-ui/components/database/DatabaseToolbar.vue +++ b/dbrepo-ui/components/database/DatabaseToolbar.vue @@ -10,21 +10,21 @@ <span v-if="database && $vuetify.display.lgAndUp" v-text="database.name" /> - <v-tooltip - v-if="database" - bottom> - <template v-slot:activator="{ props }"> - <v-icon - class="mr-2" - size="small" - right - :color="database.is_public ? 'success' : 'chip'" - v-bind="props"> - {{ database.is_public ? 'mdi-lock-open-outline' : 'mdi-lock-outline' }} - </v-icon> - </template> - <span>{{ $t('toolbars.database.' + (database.is_public ? 'public' : 'private')) }}</span> - </v-tooltip> + <v-chip + v-if="database && database.is_public" + size="small" + class="ml-2" + color="success" + :text="$t('toolbars.database.public')" + variant="outlined" /> + <v-chip + v-if="database && !database.is_public" + size="small" + class="ml-2" + :color="colorVariant" + variant="outlined" + :text="$t('toolbars.database.private')" + flat /> </v-toolbar-title> <v-spacer /> <v-btn @@ -33,6 +33,14 @@ color="tertiary" :variant="buttonVariant" :text="$t('toolbars.database.dashboard.permanent') + ($vuetify.display.lgAndUp ? ' ' + $t('toolbars.database.dashboard.xl') : '')" /> + <v-btn + v-if="canCreateTable" + :prepend-icon="$vuetify.display.lgAndUp ? 'mdi-table-large-plus' : null" + color="secondary" + variant="flat" + :text="($vuetify.display.lgAndUp ? $t('toolbars.database.create-table.xl') + ' ' : '') + $t('toolbars.database.create-table.permanent')" + class="mr-2" + :to="`/database/${$route.params.database_id}/table/create/dataset`" /> <v-btn v-if="canCreateSubset" :prepend-icon="$vuetify.display.lgAndUp ? 'mdi-wrench' : null" @@ -49,14 +57,6 @@ :text="($vuetify.display.lgAndUp ? $t('toolbars.database.create-view.xl') + ' ' : '') + $t('toolbars.database.create-view.permanent')" class="mr-2 white--text" :to="`/database/${$route.params.database_id}/view/create`" /> - <v-btn - v-if="canCreateTable" - :prepend-icon="$vuetify.display.lgAndUp ? 'mdi-table-large-plus' : null" - color="secondary" - variant="flat" - :text="($vuetify.display.lgAndUp ? $t('toolbars.database.create-table.xl') + ' ' : '') + $t('toolbars.database.create-table.permanent')" - class="mr-2" - :to="`/database/${$route.params.database_id}/table/create/dataset`" /> <v-btn v-if="canCreateIdentifier" :prepend-icon="$vuetify.display.lgAndUp ? 'mdi-identifier' : null" @@ -117,6 +117,9 @@ export default { roles () { return this.userStore.getRoles }, + colorVariant () { + return this.isContrastTheme ? '' : (this.isDarkTheme ? 'tertiary' : 'secondary') + }, canCreateIdentifier () { if (!this.roles) { return false diff --git a/dbrepo-ui/components/identifier/Citation.vue b/dbrepo-ui/components/identifier/Citation.vue index 8cd96902d4..6f35ac915e 100644 --- a/dbrepo-ui/components/identifier/Citation.vue +++ b/dbrepo-ui/components/identifier/Citation.vue @@ -10,8 +10,8 @@ <v-select v-model="style" :items="styles" - item-title="style" - item-value="accept" + item-title="title" + item-value="value" dense variant="outlined" single-line /> @@ -33,39 +33,38 @@ export default { return { loading: false, styles: [ - { style: 'APA', accept: 'text/bibliography;style=apa' }, - { style: 'IEEE', accept: 'text/bibliography;style=ieee' }, - { style: 'BibTeX', accept: 'text/bibliography;style=bibtex' } + { title: 'APA', value: 'text/bibliography;style=apa' }, + { title: 'IEEE', value: 'text/bibliography;style=ieee' }, + { title: 'BibTeX', value: 'text/bibliography;style=bibtex' } ], - style: null, + style: 'text/bibliography;style=apa', citation: null } }, watch: { style () { - this.loadCitation(this.style) + this.loadCitation() }, pid () { - this.loadCitation(this.style) + this.loadCitation() } }, mounted () { - this.style = this.styles[0].accept - this.loadCitation(null) + this.loadCitation() }, methods: { - loadCitation (accept) { - if (!this.identifier || !accept) { + loadCitation () { + if (!this.identifier || !this.style) { return } this.loading = true const identifierService = useIdentifierService() - identifierService.findOne(this.identifier.id, accept) + identifierService.findOne(this.identifier.id, this.style) .then((citation) => { this.citation = citation this.loading = false }) - .error(({code, message}) => { + .catch(({code, message}) => { const toast = useToastInstance() toast.error(this.$t(`${code}: ${message}`)) this.loading = false diff --git a/dbrepo-ui/components/subset/SubsetList.vue b/dbrepo-ui/components/subset/SubsetList.vue index 21d4e7263e..b977daffa5 100644 --- a/dbrepo-ui/components/subset/SubsetList.vue +++ b/dbrepo-ui/components/subset/SubsetList.vue @@ -1,7 +1,7 @@ <template> <div> <v-card - v-if="!loadingSubsets && queries.length === 0" + v-if="!loadingSubsets && subsets.length === 0" variant="flat" rounded="0" :text="$t('pages.database.subpages.subsets.empty')" /> @@ -14,7 +14,7 @@ <Loading /> </v-list-item> <div - v-for="(item, i) in queries" + v-for="(item, i) in subsets" :key="`q-${i}`"> <v-divider v-if="i !== 0" class="mx-4" /> <v-list> @@ -54,9 +54,7 @@ export default { return { loadingSubsets: false, loadingIdentifiers: false, - queries: [], - identifiers: [], - isAuthorizationError: false, + subsets: [], cacheStore: useCacheStore(), userStore: useUserStore() } @@ -77,8 +75,9 @@ export default { this.loadingSubsets = true const queryService = useQueryService() queryService.findAll(this.$route.params.database_id, true) - .then((queries) => { - this.queries = queries + .then((subsets) => { + this.loadingSubsets = false + this.subsets = subsets }) .catch(({code}) => { this.loadingSubsets = false @@ -88,9 +87,6 @@ export default { } toast.error(this.$t(code)) }) - .finally(() => { - this.loadingSubsets = false - }) }, title (query) { if (query.identifiers.length === 0) { diff --git a/dbrepo-ui/components/subset/SubsetToolbar.vue b/dbrepo-ui/components/subset/SubsetToolbar.vue index b51cc1d089..5c5081a2f8 100644 --- a/dbrepo-ui/components/subset/SubsetToolbar.vue +++ b/dbrepo-ui/components/subset/SubsetToolbar.vue @@ -1,6 +1,7 @@ <template> <div> - <v-toolbar flat> + <v-toolbar + flat> <v-btn class="mr-2" variant="plain" @@ -27,16 +28,16 @@ color="secondary" variant="flat" class="mb-1 ml-2" - :prepend-icon="$vuetify.display.lgAndUp ? 'mdi-content-save-outline' : null" + :prepend-icon="$vuetify.display.lgAndUp ? 'mdi-star' : null" :text="$t('toolbars.subset.save.permanent')" @click.stop="save" /> <v-btn v-if="canForgetQuery" :loading="loadingSave" - color="error" + color="warning" variant="flat" class="mb-1 ml-2" - :prepend-icon="$vuetify.display.lgAndUp ? 'mdi-trash-can-outline' : null" + :prepend-icon="$vuetify.display.lgAndUp ? 'mdi-star-off' : null" :text="$t('toolbars.subset.unsave.permanent')" @click.stop="forget" /> <v-btn @@ -116,7 +117,10 @@ export default { if (!this.database) { return false } - return this.database.is_public + if (this.database.is_public) { + return true + } + return this.access }, identifier () { /* mount pid */ @@ -154,7 +158,7 @@ export default { return formatTimestampUTCLabel(this.subset.created) }, result_visibility () { - if (!this.database) { + if (!this.database || !this.subset) { return false } if (this.database.is_public) { diff --git a/dbrepo-ui/components/view/ViewToolbar.vue b/dbrepo-ui/components/view/ViewToolbar.vue index c107c3c0ef..c43b730397 100644 --- a/dbrepo-ui/components/view/ViewToolbar.vue +++ b/dbrepo-ui/components/view/ViewToolbar.vue @@ -11,16 +11,17 @@ <v-spacer /> <v-btn v-if="canDeleteView" - prepend-icon="mdi-delete" class="mr-2" variant="flat" - color="error" - :text="$vuetify.display.lgAndUp ? $t('navigation.delete') : ''" + :prepend-icon="$vuetify.display.lgAndUp ? 'mdi-delete' : null" :loading="loadingDelete" + color="error" + :text="$t('navigation.delete')" @click="deleteView" /> <v-btn v-if="canCreatePid" - prepend-icon="mdi-content-save-outline" + class="mr-2" + :prepend-icon="$vuetify.display.lgAndUp ? 'mdi-content-save-outline' : null" variant="flat" color="primary" :text="($vuetify.display.lgAndUp ? $t('toolbars.view.pid.xl') + ' ' : '') + $t('toolbars.view.pid.permanent')" @@ -63,6 +64,10 @@ export default { database () { return this.cacheStore.getDatabase }, + buttonVariant () { + const runtimeConfig = useRuntimeConfig() + return this.$vuetify.theme.global.name.toLowerCase().endsWith('contrast') ? runtimeConfig.public.variant.button.contrast : runtimeConfig.public.variant.button.normal + }, view () { if (!this.database) { return null diff --git a/dbrepo-ui/composables/axios-instance.ts b/dbrepo-ui/composables/axios-instance.ts index a2b6ca4f6e..d4f274b717 100644 --- a/dbrepo-ui/composables/axios-instance.ts +++ b/dbrepo-ui/composables/axios-instance.ts @@ -8,7 +8,7 @@ export const useAxiosInstance = () => { const userStore = useUserStore() if (!instance) { instance = axios.create({ - timeout: 10_000, + timeout: 90_000, params: {}, headers: { Accept: 'application/json', diff --git a/dbrepo-ui/composables/identifier-service.ts b/dbrepo-ui/composables/identifier-service.ts index a85f05a45c..96e5610c8c 100644 --- a/dbrepo-ui/composables/identifier-service.ts +++ b/dbrepo-ui/composables/identifier-service.ts @@ -2,7 +2,7 @@ import type {AxiosError, AxiosRequestConfig} from 'axios' import {axiosErrorToApiError} from '@/utils' export const useIdentifierService = (): any => { - async function findOne(id: number, accept: string | null): Promise<IdentifierDto> { + async function findOne(id: number, accept: string): Promise<IdentifierDto> { const axios = useAxiosInstance() console.debug('find identifier with id', id) const config: AxiosRequestConfig = { diff --git a/dbrepo-ui/composables/query-service.ts b/dbrepo-ui/composables/query-service.ts index fa18a55ec0..abcd928b66 100644 --- a/dbrepo-ui/composables/query-service.ts +++ b/dbrepo-ui/composables/query-service.ts @@ -13,6 +13,10 @@ export const useQueryService = (): any => { resolve(response.data) }) .catch((error) => { + if (error.response.status === 403) { + /* ignore */ + resolve([]) + } console.error('Failed to find queries', error) reject(axiosErrorToApiError(error)) }) @@ -77,7 +81,7 @@ export const useQueryService = (): any => { const axios = useAxiosInstance() console.debug('execute query in database with id', databaseId) return new Promise<QueryResultDto>((resolve, reject) => { - axios.post<QueryResultDto>(`/api/database/${databaseId}/subset`, data, {params: mapFilter(timestamp, page, size)}) + axios.post<QueryResultDto>(`/api/database/${databaseId}/subset`, data, {params: mapFilter(timestamp, page, size), timeout: 600_000}) .then((response) => { console.info('Executed query with id', response.data.id, ' in database with id', databaseId) resolve(response.data) @@ -93,7 +97,7 @@ export const useQueryService = (): any => { const axios = useAxiosInstance() console.debug('re-execute query in database with id', databaseId) return new Promise<QueryResultDto>((resolve, reject) => { - axios.get<QueryResultDto>(`/api/database/${databaseId}/subset/${queryId}/data`, { params: mapFilter(null, page, size), timeout: 30_000 }) + axios.get<QueryResultDto>(`/api/database/${databaseId}/subset/${queryId}/data`, { params: mapFilter(null, page, size) }) .then((response) => { console.info('Re-executed query in database with id', databaseId) resolve(response.data) @@ -109,7 +113,7 @@ export const useQueryService = (): any => { const axios = useAxiosInstance() console.debug('re-execute query in database with id', databaseId) return new Promise<number>((resolve, reject) => { - axios.head<void>(`/api/database/${databaseId}/subset/${queryId}/data`, { timeout: 30_000 }) + axios.head<void>(`/api/database/${databaseId}/subset/${queryId}/data`) .then((response) => { const count: number = Number(response.headers['x-count']) console.info('Found', count, 'tuples for query', queryId, 'in database with id', databaseId) diff --git a/dbrepo-ui/composables/table-service.ts b/dbrepo-ui/composables/table-service.ts index 88ab27b39e..ffd7ebcd60 100644 --- a/dbrepo-ui/composables/table-service.ts +++ b/dbrepo-ui/composables/table-service.ts @@ -71,7 +71,7 @@ export const useTableService = (): any => { const axios = useAxiosInstance() console.debug('get data for table with id', tableId, 'in database with id', databaseId); return new Promise<QueryResultDto>((resolve, reject) => { - axios.get<QueryResultDto>(`/api/database/${databaseId}/table/${tableId}/data`, { params: mapFilter(timestamp, page, size), timeout: 30_000 }) + axios.get<QueryResultDto>(`/api/database/${databaseId}/table/${tableId}/data`, { params: mapFilter(timestamp, page, size) }) .then((response) => { console.info('Got data for table with id', tableId, 'in database with id', databaseId) resolve(response.data) @@ -87,7 +87,7 @@ export const useTableService = (): any => { const axios = useAxiosInstance() console.debug('get data count for table with id', tableId, 'in database with id', databaseId); return new Promise<number>((resolve, reject) => { - axios.head<void>(`/api/database/${databaseId}/table/${tableId}/data`, { params: mapFilter(timestamp, null, null), timeout: 30_000 }) + axios.head<void>(`/api/database/${databaseId}/table/${tableId}/data`, { params: mapFilter(timestamp, null, null) }) .then((response: AxiosResponse<void>) => { const count: number = Number(response.headers['x-count']) console.info('Found' + count + 'in table with id', tableId, 'in database with id', databaseId) @@ -191,7 +191,7 @@ export const useTableService = (): any => { const axios = useAxiosInstance() console.debug('suggest semantic entities for table column with id', columnId, 'of table with id', tableId, 'of database with id', databaseId) return new Promise<TableColumnEntityDto[]>((resolve, reject) => { - axios.get<TableColumnEntityDto[]>(`/api/database/${databaseId}/table/${tableId}/column/${columnId}/suggest`, {timeout: 10000}) + axios.get<TableColumnEntityDto[]>(`/api/database/${databaseId}/table/${tableId}/column/${columnId}/suggest`) .then((response) => { console.info('Suggested semantic entities for table column with id', columnId, 'of table with id', tableId, 'of database with id', databaseId) resolve(response.data) diff --git a/dbrepo-ui/composables/view-service.ts b/dbrepo-ui/composables/view-service.ts index adc76e32a1..642a7c6e51 100644 --- a/dbrepo-ui/composables/view-service.ts +++ b/dbrepo-ui/composables/view-service.ts @@ -37,7 +37,7 @@ export const useViewService = (): any => { const axios = useAxiosInstance() console.debug('re-execute view with id', viewId, 'in database with id', databaseId) return new Promise<QueryResultDto>((resolve, reject) => { - axios.get<QueryResultDto>(`/api/database/${databaseId}/view/${viewId}/data`, { params: {page, size}, timeout: 30_000 }) + axios.get<QueryResultDto>(`/api/database/${databaseId}/view/${viewId}/data`, { params: {page, size} }) .then((response) => { console.info('Re-executed view with id', viewId, 'in database with id', databaseId) resolve(response.data) @@ -53,7 +53,7 @@ export const useViewService = (): any => { const axios = useAxiosInstance() console.debug('re-execute view with id', viewId, 'in database with id', databaseId) return new Promise<number>((resolve, reject) => { - axios.head<number>(`/api/database/${databaseId}/view/${viewId}/data`, { timeout: 30_000 }) + axios.head<number>(`/api/database/${databaseId}/view/${viewId}/data`) .then((response) => { const count: number = Number(response.headers['x-count']) console.info('Found', count, 'tuples for view with id', viewId, 'in database with id', databaseId) diff --git a/dbrepo-ui/locales/en-US.json b/dbrepo-ui/locales/en-US.json index cf8dcfd03e..be8ddfcc23 100644 --- a/dbrepo-ui/locales/en-US.json +++ b/dbrepo-ui/locales/en-US.json @@ -1382,7 +1382,7 @@ }, "subset": { "save": { - "permanent": "Save" + "permanent": "Star" }, "export": { "data": { @@ -1399,7 +1399,7 @@ "permanent": "PID" }, "unsave": { - "permanent": "Unsave" + "permanent": "Unstar" } }, "view": { diff --git a/dbrepo-ui/pages/database/[database_id]/settings.vue b/dbrepo-ui/pages/database/[database_id]/settings.vue index 61e9dc6244..a79ad48c41 100644 --- a/dbrepo-ui/pages/database/[database_id]/settings.vue +++ b/dbrepo-ui/pages/database/[database_id]/settings.vue @@ -422,7 +422,7 @@ export default { databaseService.updateVisibility(this.$route.params.database_id, this.modifyVisibility) .then((database) => { const toast = useToastInstance() - toast.success('success.database.visibility') + toast.success(this.$t('success.database.visibility')) this.cacheStore.setDatabase(database) }) .catch(() => { diff --git a/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/data.vue b/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/data.vue index f740416faa..4902e2c54f 100644 --- a/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/data.vue +++ b/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/data.vue @@ -14,6 +14,15 @@ v-else v-text="executionUTC" /> </v-toolbar-title> + <v-spacer /> + <v-btn + :prepend-icon="$vuetify.display.lgAndUp ? 'mdi-refresh' : null" + variant="flat" + :text="$t('toolbars.table.data.refresh')" + class="mr-2" + :disabled="loadingSubset" + :loading="loadingSubset" + @click="loadSubset" /> </v-toolbar> <v-card tile> <QueryResults diff --git a/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/info.vue b/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/info.vue index d52ac91642..1d9101fbf2 100644 --- a/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/info.vue +++ b/dbrepo-ui/pages/database/[database_id]/subset/[subset_id]/info.vue @@ -103,7 +103,12 @@ <script setup> const config = useRuntimeConfig() const { database_id, subset_id } = useRoute().params -const { data } = await useFetch(`${config.public.api.server}/api/database/${database_id}/subset/${subset_id}`) +const requestConfig = { timeout: 90_000, headers: { Accept: 'application/json', 'Content-Type': 'application/json' } } +const userStore = useUserStore() +if (userStore.getToken) { + requestConfig.headers.Authorization = `Bearer ${userStore.getToken}` +} +const { data } = await useFetch(`${config.public.api.server}/api/database/${database_id}/subset/${subset_id}`, requestConfig) if (data.value) { const identifierService = useIdentifierService() useServerHead(identifierService.subsetToServerHead(data.value)) diff --git a/dbrepo-ui/pages/database/[database_id]/table/[table_id]/data.vue b/dbrepo-ui/pages/database/[database_id]/table/[table_id]/data.vue index 2d7195fbcd..bcab9b60be 100644 --- a/dbrepo-ui/pages/database/[database_id]/table/[table_id]/data.vue +++ b/dbrepo-ui/pages/database/[database_id]/table/[table_id]/data.vue @@ -72,7 +72,7 @@ :headers="headers" :items="rows" :items-length="total" - :loading="loadingData" + :loading="loadingData || loadingCount" :options.sync="options" :footer-props="footerProps" @update:options="loadData"> @@ -192,9 +192,6 @@ export default { } }, computed: { - loadingColor () { - return this.error ? 'error' : 'primary' - }, roles () { return this.userStore.getRoles }, @@ -283,6 +280,7 @@ export default { }, watch: { version () { + this.loadCount() this.reload() }, table (newTable, oldTable) { @@ -292,8 +290,8 @@ export default { } }, mounted () { - this.reload() this.loadProperties() + this.loadCount() }, methods: { addTuple () { @@ -429,9 +427,20 @@ export default { reload () { this.lastReload = new Date() this.loadData({ page: this.options.page, itemsPerPage: this.options.itemsPerPage, sortBy: null}) - this.loadCount() }, - loadData ({ page, itemsPerPage, sortBy }) { + loadCount() { + this.loadingCount = true + const tableService = useTableService() + tableService.getCount(this.$route.params.database_id, this.$route.params.table_id, (this.versionISO || this.lastReload.toISOString())) + .then((count) => { + this.total = count + this.loadingCount = false + }) + .catch((error) => { + this.loadingCount = false + }) + }, + loadData({ page, itemsPerPage, sortBy }) { this.options.page = page this.options.itemsPerPage = itemsPerPage const tableService = useTableService() @@ -464,23 +473,6 @@ export default { toast.error(this.$t(code) + ": " + message) }) }, - loadCount () { - const tableService = useTableService() - this.loadingCount = true - tableService.getCount(this.$route.params.database_id, this.$route.params.table_id, (this.versionISO || this.lastReload.toISOString())) - .then((count) => { - this.total = count - this.loadingCount = false - }) - .catch(({code, message}) => { - this.loadingCount = false - const toast = useToastInstance() - if (typeof code !== 'string' || typeof message !== 'string') { - return - } - toast.error(this.$t(code) + ": " + message) - }) - }, isFileField (column) { return ['blob', 'longblob', 'mediumblob', 'tinyblob'].includes(column.column_type) }, diff --git a/dbrepo-ui/pages/database/[database_id]/view/[view_id]/data.vue b/dbrepo-ui/pages/database/[database_id]/view/[view_id]/data.vue index 03464f5dbb..838ef2f0f1 100644 --- a/dbrepo-ui/pages/database/[database_id]/view/[view_id]/data.vue +++ b/dbrepo-ui/pages/database/[database_id]/view/[view_id]/data.vue @@ -10,7 +10,7 @@ :prepend-icon="$vuetify.display.lgAndUp ? 'mdi-refresh' : null" variant="flat" :text="$t('toolbars.table.data.refresh')" - class="mb-1 ml-2" + class="mb-1 mr-2" :loading="loadingData" @click="reload" /> </v-toolbar> diff --git a/dbrepo-ui/utils/index.ts b/dbrepo-ui/utils/index.ts index d940d076ae..995c03a827 100644 --- a/dbrepo-ui/utils/index.ts +++ b/dbrepo-ui/utils/index.ts @@ -1082,28 +1082,30 @@ export function timestampsToHumanDifference(date1: string, date2: string) { return moment.duration(other.diff(date)).humanize(true) } -export function sizeToHumanLabel(num: number) { +export function sizeToHumanLabel(num: number): string { let number = Number(num) if (!number) { return '0 B' } if (number < 1000) { - return `${Math.floor(number)} B` + return `${roundTwoDecimals(number)} B` } number = number / 1000 if (number < 1000) { - return `${Math.floor(number)} kB` + return `${roundTwoDecimals(number)} kB` } number = number / 1000 if (number < 1000) { - return `${Math.floor(number)} MB` + return `${roundTwoDecimals(number)} MB` } number = number / 1000 if (number < 1000) { - return `${number} GB` + return `${roundTwoDecimals(number)} GB` } number = number / 1000 - if (number < 1000) { - return `${number} TB` - } + return `${roundTwoDecimals(number)} TB` +} + +export function roundTwoDecimals(num: number): number { + return Math.round((num + Number.EPSILON) * 100) / 100 } diff --git a/helm/dbrepo/templates/auth-configmap.yaml b/helm/dbrepo/templates/auth-configmap.yaml index 0ef8e90bf9..269d18c99d 100644 --- a/helm/dbrepo/templates/auth-configmap.yaml +++ b/helm/dbrepo/templates/auth-configmap.yaml @@ -2184,7 +2184,7 @@ data: } ], "org.keycloak.storage.UserStorageProvider" : [ { "id" : "c109d473-5ce1-4032-af7b-02e5442f5c07", - "name" : "openldap", + "name" : "Identity Service", "providerId" : "ldap", "subComponents" : { "org.keycloak.storage.ldap.mappers.LDAPStorageMapper" : [ { @@ -2944,4 +2944,4 @@ data: "policies" : [ ] } } -{{- end }} \ No newline at end of file +{{- end }} diff --git a/helm/dbrepo/templates/metadata-secret.yaml b/helm/dbrepo/templates/metadata-secret.yaml index ac9fbb32fe..0718b02ea1 100644 --- a/helm/dbrepo/templates/metadata-secret.yaml +++ b/helm/dbrepo/templates/metadata-secret.yaml @@ -36,7 +36,7 @@ stringData: METADATA_HOST: "{{ .Values.metadatadb.host }}" METADATA_JDBC_EXTRA_ARGS: "{{ .Values.metadatadb.jdbcExtraArgs }}" METADATA_USERNAME: "{{ .Values.metadatadb.rootUser.user }}" - METADATA_PASSWORD: "{{ .Values.metadatadb.rootUser.password }}" + METADATA_DB_PASSWORD: "{{ .Values.metadatadb.rootUser.password }}" PID_BASE: "{{ $pidBase }}" REPOSITORY_NAME: "{{ .Values.metadataservice.repositoryName }}" ROR_ENDPOINT: "{{ .Values.metadataservice.ror.endpoint}}" diff --git a/helm/dbrepo/templates/search-db-secret.yaml b/helm/dbrepo/templates/search-db-secret.yaml deleted file mode 100644 index 81cc79db7c..0000000000 --- a/helm/dbrepo/templates/search-db-secret.yaml +++ /dev/null @@ -1,79 +0,0 @@ ---- -apiVersion: v1 -kind: Secret -type: kubernetes.io/tls -metadata: - name: search-db-secret - namespace: {{ .Values.namespace }} -data: - tls.crt: | - LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM0akNDQWNxZ0F3SUJBZ0lSQVAvcFJoaFQ5 - SFVWaUFzYitybmJjdkV3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSmMyVmhj - bU5vTFdSaU1CNFhEVEkwTURRd056RTRORFEwT0ZvWERUSTBNRGN3TmpFNApORFEwT0Zvd0ZERVNN - QkFHQTFVRUF4TUpjMlZoY21Ob0xXUmlNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFN - SUlCQ2dLQ0FRRUFzb3lTWTduV3J0MTRVQjNST0kwOWRtMVNSU2lDZ2hQYVhwRlJTMjhjalpNSUFz - TUoKR2ZwREZ5VktOQ3pTV0VZN0J2M1JpOHlrRnlZNkpFb2p0S3oxdk9GNnNyQ2JYZnhsY1NiZDk3 - UVYwdU9IYTZKRApsWGN2aUJEKzN2ZTB0K0MzRGFPSFVMY1liVWkzS2xOS3FwTDU1Q2ZNeTYzdU4z - a21zekRwTjVycWhOYnBlVTAxCnd2NFZNaldNZ3RlU1VpWDNqeU1EcUFOa1B3UXFiYnZHN0hBUm54 - Q0QvMHJFeEVvRjNqRCtGV01XbEVjdXR1VGkKbFJ5QnN3L1FLTWd0aVJVSFJXYUJGK2ZES0wxSUoz - YVhmcDR5bmNhL2tCR3pxVGpqb3dJb2R0MEdOZjBFa1QyQgpTWG9hZGtwdVptT2JiVDF2UmN1T1BH - UEZjWVd5Qm1ucS9adEorUUlEQVFBQm95OHdMVEFkQmdOVkhTVUVGakFVCkJnZ3JCZ0VGQlFjREFR - WUlLd1lCQlFVSEF3SXdEQVlEVlIwVEFRSC9CQUl3QURBTkJna3Foa2lHOXcwQkFRc0YKQUFPQ0FR - RUFlUU4vaUsvRzhHbGt5R0w1NjlrZnBiWEE2bE8vRHFObGlXRkgrY2ZIZ0NzYWxKMWVSSjliY1RZ - dgo0S3Y0MDlWUWpCbVg0WTRqMUt6R1ZnYkZaZkh1Ry9Nb0dzWVVnQ1VjTm94ZThtM0ZUcjRwYnZT - MXNUV0V4cGFNCkpSMURQQmNMV0o3MndTQzBkRFpISC9hVVNSMUs4UGpnMWtaMVRINTdvZDJoNWpJ - RUFhZkd1ZGhzejVpWlZQcVkKR1lrakZhRklVeXpjWkxUbjFBNXRwSlpTRmhxZHZGQmFndURUYkp4 - NmROVWZVc0sxZXFuaThSQVN6L3dPbHQwcQpSckExbVdCTEI1NW9XRzh4ZXZicmtNNUNuSWVvL2hS - SG83cE1pUFQxWE5uT2cvNjhmZEc4T0lXMFFhNjdMVEZnCnU2dTkxQ1BmVk5KVHQ5bmlZWHJ4N1hl - SEJ2dW1iUT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K - ca.crt: | - LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM0akNDQWNxZ0F3SUJBZ0lSQVAvcFJoaFQ5 - SFVWaUFzYitybmJjdkV3RFFZSktvWklodmNOQVFFTEJRQXcKRkRFU01CQUdBMVVFQXhNSmMyVmhj - bU5vTFdSaU1CNFhEVEkwTURRd056RTRORFEwT0ZvWERUSTBNRGN3TmpFNApORFEwT0Zvd0ZERVNN - QkFHQTFVRUF4TUpjMlZoY21Ob0xXUmlNSUlCSWpBTkJna3Foa2lHOXcwQkFRRUZBQU9DCkFROEFN - SUlCQ2dLQ0FRRUFzb3lTWTduV3J0MTRVQjNST0kwOWRtMVNSU2lDZ2hQYVhwRlJTMjhjalpNSUFz - TUoKR2ZwREZ5VktOQ3pTV0VZN0J2M1JpOHlrRnlZNkpFb2p0S3oxdk9GNnNyQ2JYZnhsY1NiZDk3 - UVYwdU9IYTZKRApsWGN2aUJEKzN2ZTB0K0MzRGFPSFVMY1liVWkzS2xOS3FwTDU1Q2ZNeTYzdU4z - a21zekRwTjVycWhOYnBlVTAxCnd2NFZNaldNZ3RlU1VpWDNqeU1EcUFOa1B3UXFiYnZHN0hBUm54 - Q0QvMHJFeEVvRjNqRCtGV01XbEVjdXR1VGkKbFJ5QnN3L1FLTWd0aVJVSFJXYUJGK2ZES0wxSUoz - YVhmcDR5bmNhL2tCR3pxVGpqb3dJb2R0MEdOZjBFa1QyQgpTWG9hZGtwdVptT2JiVDF2UmN1T1BH - UEZjWVd5Qm1ucS9adEorUUlEQVFBQm95OHdMVEFkQmdOVkhTVUVGakFVCkJnZ3JCZ0VGQlFjREFR - WUlLd1lCQlFVSEF3SXdEQVlEVlIwVEFRSC9CQUl3QURBTkJna3Foa2lHOXcwQkFRc0YKQUFPQ0FR - RUFlUU4vaUsvRzhHbGt5R0w1NjlrZnBiWEE2bE8vRHFObGlXRkgrY2ZIZ0NzYWxKMWVSSjliY1RZ - dgo0S3Y0MDlWUWpCbVg0WTRqMUt6R1ZnYkZaZkh1Ry9Nb0dzWVVnQ1VjTm94ZThtM0ZUcjRwYnZT - MXNUV0V4cGFNCkpSMURQQmNMV0o3MndTQzBkRFpISC9hVVNSMUs4UGpnMWtaMVRINTdvZDJoNWpJ - RUFhZkd1ZGhzejVpWlZQcVkKR1lrakZhRklVeXpjWkxUbjFBNXRwSlpTRmhxZHZGQmFndURUYkp4 - NmROVWZVc0sxZXFuaThSQVN6L3dPbHQwcQpSckExbVdCTEI1NW9XRzh4ZXZicmtNNUNuSWVvL2hS - SG83cE1pUFQxWE5uT2cvNjhmZEc4T0lXMFFhNjdMVEZnCnU2dTkxQ1BmVk5KVHQ5bmlZWHJ4N1hl - SEJ2dW1iUT09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K - tls.key: | - LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV2UUlCQURBTkJna3Foa2lHOXcwQkFRRUZB - QVNDQktjd2dnU2pBZ0VBQW9JQkFRQ3lqSkpqdWRhdTNYaFEKSGRFNGpUMTJiVkpGS0lLQ0U5cGVr - VkZMYnh5Tmt3Z0N3d2taK2tNWEpVbzBMTkpZUmpzRy9kR0x6S1FYSmpvawpTaU8wclBXODRYcXlz - SnRkL0dWeEp0MzN0QlhTNDRkcm9rT1ZkeStJRVA3ZTk3UzM0TGNObzRkUXR4aHRTTGNxClUwcXFr - dm5rSjh6THJlNDNlU2F6TU9rM211cUUxdWw1VFRYQy9oVXlOWXlDMTVKU0pmZVBJd09vQTJRL0JD - cHQKdThic2NCR2ZFSVAvU3NURVNnWGVNUDRWWXhhVVJ5NjI1T0tWSElHekQ5QW95QzJKRlFkRlpv - RVg1OE1vdlVnbgpkcGQrbmpLZHhyK1FFYk9wT09PakFpaDIzUVkxL1FTUlBZRkplaHAyU201bVk1 - dHRQVzlGeTQ0OFk4VnhoYklHCmFlcjltMG41QWdNQkFBRUNnZ0VCQUtPZ3A5ZTB5OFhkT1JGVEFo - WXRlaEk2QlpkVGxLYll3dHEvbWh6amF1dGoKdjRlb2JZTGRFdmIzT1pXdkxlV3dGeEJGTS9CR1Rt - cllvWmY0U2RpZVdXWUx6WUpNejFYR3BNQ1p1Zm56azd4OAp2L0luOW4vWGhqdlFONExteHp0c09O - WEs4NHRKQUozR2NmWGI5eVZ6SklldTRjUVhWYVNJNXFwNVBJRzArdzlZCnk2NTFWZkZJQUd3SmRI - QlpId1lmQUdxbU5oVlo3MDc4TVUxQWU2Y2VkZjJ0RnlWYW5ScXBLUFZ1Z0tGQy9kRG8KVXJIMHRJ - ajFkU3RKRGxucHJ3YVYrMDRkUDBvZnlBc09ablp3VXRzZE8vM1ZMMWR4bCtIT1dGeUpvSjI1dkF2 - eAp5ZW5qc0dzd1pJRW1oUzM0NVRVTlFNbTJzYnJtYklMS1dpWEp5SmlEeWdFQ2dZRUF4QUNEbWxG - ZmsxZDlkSmJTCi90NDRGTzFiUVQ2b1Z2VWJ2NGMwcnRLVVhwTnFlajNmbXl4bUJINm82SkhoYjNO - eXdWa3U0QW9YTXBFTkhISDAKNlN2TzBYM2U1MU8xOXppQ25hMGdoZExSS3JIS3ZqbzRNdmdiV3Ey - Z3NJNmpJQkxTcU0zNXNxTjhGRXRVZko4TQpKRUZIMThJTThTRjEyQ2hWYjVWaThTTDgvY2tDZ1lF - QTZUUmhVQi9nQVM1RmN5V1NHNGR4UWtHWG03R1lSamFrCjJVWUlOUUIyV2d2Qk9vN2tvV21nR1M0 - eE9YanJsZ2NrMmhsTEhZSXAycnRoSWdNMUdBQUdqa2lYeXJPVE9kaGQKeUJ0RjBMS0kvVjlBZUxK - eExkRDBYalB0WThIYXdTcW8weGdxUml2RzlJUFBlSGZ6SmlraXV6enNOT3pjVlk1aApkZktqZy9J - eVFyRUNnWUJ1c0dlaDk4Q0ZBa3pNVWZ6NGlHQ2RtT29ITDY1NzVWSjFXSkx0QStsY2U5NFBDUEJG - CnZzNGlUYkZ3SGlwMCtYcmVMRkpubmVzNTJHYlNJSjBTTFhaUUlzaUdWV1VYSjZmRUNpaXF5c0xy - WEpyRjBUVTUKdTVvZkhKejUrS094RWxBN21vOGdUbWxkUUttRzgzODAzbFVIU1FSc0ROeHpaVnZT - ZDBmNExDMDUyUUtCZ0h0YQpzNmJZVlhzS2FMNFJ2NGxFU1lxTWU0OWxqM0NFY3dwaTJ2QitRQnc5 - WDRhRUV6ZTJVWE5BVmRWYXV2THU4SFZWCkw4QjZHMzJSNUQxRGlSQWE0MXpiMVQ3cFloVVU5L1pq - UnJpdjEzcCtxZkd1SWVQa1JYNlc1UmtCYjU4QjI2OWQKZHU4TE5RQWR3TjZ1UkRXSlNNL1YxL1Bl - M21WN0hONXc3RUZkR1d6aEFvR0FiN29HOWhoTGxTL212bEhHVE11egp6SXhtSTBaTisyZGFqNEYy - MDZHRGJYaUkwYUtFTHlteVJkcEZYcStzaWpaTzFqdXU4WDhMS0FCOUVaeEtXRzRICjNnQWoxWGFF - QXFjNjFkaFkveHRHUDR2OXN3UElmVjR3NGtTUHBWYlFFd3ZIcko4K1V2NjhuOTZTU3dkK1BBNisK - OFFwUEVYSXlscXB4UlFyMnN5amhoOTQ9Ci0tLS0tRU5EIFBSSVZBVEUgS0VZLS0tLS0K \ No newline at end of file diff --git a/helm/dbrepo/templates/search-secret.yaml b/helm/dbrepo/templates/search-secret.yaml index 41665ac2bc..251da00248 100644 --- a/helm/dbrepo/templates/search-secret.yaml +++ b/helm/dbrepo/templates/search-secret.yaml @@ -13,7 +13,7 @@ stringData: AUTH_SERVICE_CLIENT: "{{ .Values.authservice.client.id }}" AUTH_SERVICE_CLIENT_SECRET: "{{ .Values.authservice.client.secret }}" AUTH_SERVICE_ENDPOINT: "{{ .Values.authservice.endpoint }}" - GATEWAY_SERVICE_ENDPOINT: "{{ .Values.gateway }}" + METADATA_SERVICE_ENDPOINT: "{{ .Values.metadataservice.endpoint }}" JWT_PUBKEY: "{{ .Values.authservice.jwt.pubkey }}" LOG_LEVEL: "{{ ternary "DEBUG" "INFO" .Values.searchservice.image.debug }}" OPENSEARCH_HOST: "{{ .Values.searchdb.host }}" diff --git a/helm/dbrepo/values.yaml b/helm/dbrepo/values.yaml index 2b18ed1422..af810436af 100644 --- a/helm/dbrepo/values.yaml +++ b/helm/dbrepo/values.yaml @@ -56,8 +56,8 @@ metadatadb: enabled: false ## @skip metadatadb.initdbScriptsConfigMap The initial database scripts. initdbScriptsConfigMap: metadata-db-setup - ## @param metadatadb.initdbScripts Additional init.db scripts that are executed on the first start. - initdbScripts: { } + ## @param metadatadb.extraInitDbScripts Additional init.db scripts that are executed on the first start. + extraInitDbScripts: { } # 03-additional-data.sql: | # BEGIN; # INSERT INTO `mdb_containers` (name, internal_name, image_id, host, port, sidecar_host, sidecar_port, privileged_username, privileged_password) @@ -81,11 +81,6 @@ authservice: debug: false ## @param authservice.endpoint The hostname for the microservices. endpoint: http://auth-service - auth: - ## @param authservice.auth.adminUser The admin username. - adminUser: fda - ## @param authservice.auth.adminPassword The admin user password. - adminPassword: fda ## @skip authservice.postgresql postgresql: enabled: true @@ -222,8 +217,6 @@ searchdb: ## @skip searchdb.security security: enabled: false - adminUsername: admin - adminPassword: admin ## @param searchdb.clusterName The cluster name. clusterName: search-db @@ -422,7 +415,7 @@ metadataservice: ## @param metadataservice.podSecurityContext.fsGroup Set RabbitMQ pod's Security Context fsGroup fsGroup: 1001 containerSecurityContext: - ## @param metadataservice.containerSecurityContext.enabled Enabled containers' Security Context + ## @param metadataservice.containerSecurityContext.enabled Enable containers' Security Context enabled: true ## @param metadataservice.containerSecurityContext.seLinuxOptions Set SELinux options in container seLinuxOptions: { } @@ -812,14 +805,14 @@ ui: host: example.com port: ## @param ui.public.broker.port.5671 Enable display of the broker 5671 port and mark it as secure (SSL/TLS). - 5671: true + 5671: false ## @param ui.public.broker.port.5672 Enable display of the broker 5672 port and mark it as insecure (no SSL/TLS). - 5672: false + 5672: true ## @param ui.public.broker.extra Extra metadata displayed. extra: "" database: ## @param ui.public.database.extra Extra metadata displayed. - extra: "128.130.0.0/15" + extra: "" ## @skip ui.public.links links: rabbitmq: diff --git a/install.sh b/install.sh index a9fcf6d10d..5e367f4d53 100644 --- a/install.sh +++ b/install.sh @@ -69,15 +69,15 @@ fi echo "[📦] Pulling images for version ${VERSION} ..." docker compose pull -echo "[✨] Starting DBRepo ..." -docker compose up -d - -if [ $? -eq 0 ]; then - echo "[🎉] Successfully started!" - echo "" - echo "You can now inspect the logs with:" - echo "" - echo " docker compose logs -f" - echo "" - echo "Read about next steps online: https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/${VERSION}/installation/#next-steps" -fi +echo "[🎉] Success!" +echo "" +echo "You can now:" +echo "" +echo " 1) Either start the deployment running on http://localhost, or" +echo " 2) Edit the BASE_URL variable in .env to set your hostname" +echo "" +echo "Then start the local deployment with:" +echo "" +echo " docker compose up -d" +echo "" +echo "Read about next steps online: https://www.ifs.tuwien.ac.at/infrastructures/dbrepo/${VERSION}/installation/#next-steps" diff --git a/mkdocs.yml b/mkdocs.yml index 0ba1317c69..5285953568 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -47,6 +47,7 @@ nav: - Air Quality Data: examples/air.md - COVID-19 Data: examples/covid-19.md - Hazard Data: examples/hazard.md + - Health Data: examples/health.md - Industry 4.0 Power Data: examples/power.md - Survey Data: examples/survey.md - Lute Data: examples/lute-data.md -- GitLab