From f64f49a7a6eec45c7d06cd7b7474b8858422643a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BD=D1=82=D0=BE=D0=BD?= Date: Fri, 9 Jan 2026 17:50:45 +0300 Subject: [PATCH] performance improvements, charts improvements, minor UI improvements --- .../openvpn_api_v3.cpython-314.pyc | Bin 27432 -> 32241 bytes APP/openvpn_api_v3.py | 205 +++++++++++---- DOCS/api_v3_endpoints.md | 239 +++++++++--------- DOCS/data_gathering_report.md | 45 ++++ UI/client/src/App.vue | 10 +- UI/client/src/components/HistoryModal.vue | 15 +- UI/client/src/views/Analytics.vue | 2 +- 7 files changed, 337 insertions(+), 179 deletions(-) create mode 100644 DOCS/data_gathering_report.md diff --git a/APP/__pycache__/openvpn_api_v3.cpython-314.pyc b/APP/__pycache__/openvpn_api_v3.cpython-314.pyc index dac339367553fae7b42d2491982f0c79438f6790..6a1492f280326ebed52c645d6830e76ffe7d5456 100644 GIT binary patch delta 8772 zcma)C32km4wQLa|T1s&u67&V=-i(t>X& zKCSUUvr!62Ip@VEm5o#}en#n%mO%+#mYSAcRukF_v9e)W zhk9Q@h?S2NW*KN$-be-q533lK2K4yx0=-p{D{~_|8Zn4`gaRgfL#D@j%k;Y8O%YZ( zi~=^opq!-g-Yrhva9%~W?t~8^2O7=`QZ=?*a|jJ9_nbgz!Y=S+xVzjS1rZt-7a3G; z)b~k{K!IY?s&A`R)v#)_RY*8ckQ!D7>0yJi3)M2#}^}%pRYs0b< zNT5y~gIL|4`EvU9P>xiA-o3O)m_vs4^^1d_ZY z8jb;pal9lnF~Na(5ODa?jQS>*PgCyJ7^2qvt7n-%>$sV25>2{JtEyyza~fP07OHRZU$+TBx<&wb-6hd_l2psa&>H zE?QSC4M|x>j%==Vt8dxrTesFOTWc31E7m9V18 zVR*6kYom7zjTw0yl~k;I_bhw&T(_=x_b2;)ZuV|CN*0c#9o~h$j7-vEo0sPLTDCM~ zl*sDM=IwVZP50EuQgY9U+`jv}ki)YeUH9x-_Uu~sG%b6YRy-|t?0XV@=>prGf~K^= zlI{MH+lJluWXQ2MqwGXFYf5Q(=YAnF*zY4sYyFwdl99ms4^c9wKlJug_Uh;#Yb=D8 zl>&WPr|$L0UUpMJmp*URD5y&Oyr+hW;grWTv>6n*5L7220baaxe0FArdx|9GcTm?4$aZMuu#}Srf?K(kAZ`9U%nVZ*dn$Nca|5&^;vlmeDov5 zxpqfNaw(>W(Kpg!E*Gqzoopy#4XmCqu*Pkc_&KozYyn2bphL%?eLv9$CK)94!Se0; zF#4i1bRX;>B&x9{5Re4Q@K*|L`K*~Cf0`NyY+#MYgLw%3VW?y*IfYm& zV;#xMu}|Kuk>%0|1uO7u(AZ~VY^;3*wg&NGw7I$vdB;IH2V>uAfzbpjS|ARSr-t}v z_F?UeeQP>=PR7ZY_5x4bRv2TTIw!gHW=xDTx62$thzODSjF&UcVC5(Z`ruauzv@xs zm*td#9SNc}F-x$9F$Qacbtk{H>1)E~5cCgF!Cf_5{h;9Pk=k6AG3Htd)-wgc21e(X z`e{ZM@aH7(Hw%k!#wNv?az#YaFBdlqkGP4@{l9o1-zDbfXgACzTNnW69%c8`r0`g}ry7K(vO7`( zQ!Pqb_jJGYF|hBYVSKf^wih;_FxfgQD`A(M6jBvs(TW<++;knuMPs3Jtk2{3{6XsT zRCz*rSSU4~in*D{g)loiK3(DSgrXjNtJ-aQ_}D-t)H-u97LJZ_;qh?fe0V}+|7Ufn zX$LzByQj!HRRy)39;wjGy-o`b)R+_lJts(0xVX!T|E8)6A9m~T^EJtR2ZKY$0-k** zJ^ydDFZJ%!7N4%RR%0Eqnp2E?(_EL2u z3d5S3hFOjB%E(MC%mMP@HS8>;qhsTA?YOfzvtKsaLphEh#N zW}X32035P=0Ct!yi9>0-BhkN6V4sh#7SyHn#S0Iw>MPUU@?>AyU7S3WE-Am< zpEi~(L|2Vf|9i1_n{;f|Se=z7#V2yDJWyUzmhAtx0$YZrot+zD=%) zt^0d!`+HVQ`+bf69Jz!&;U&0(xyY-AUD(-Tf=Bfg76Z`uXTW+nU>dO@G_nctev^ zr(DF6wzzlQTD@$oUa{6D+1U03(amQMd>`5E!S1(?(th!s*46Vt7X=mxWvv%28 zo2u(saqdg@ZkU}5_EmFD+5$M!w`}p9UaY=0xmMM2$I`ixD@!{n7ENhK&0;7GvtFEb zG+(FEj{0jo8MVCH_B$lE+V2%0TW!XJ3eEGXw9b?^lr22*a@&g+Qq1sL#qqSk`@;T3 z)$>DmV0SS+Aj9l#gN@hd-g|qLM&|w0yS)AAyKmTwO$p_e<%K>yL$qlMQ^Z_%Jzdb800=~tAl!BXif}~Ty9-EpoqDKM^7`KLp(e>2ZVJ-oH$2jz z4)jKo3^H%DN=Uk+C{VC8TK}{(z7?BD)ZzY)5Fxm1#uW%4u)-&{g^k@rNQfOGOyAZ{}b@G@AsvFQ5&WRT-^Q(-*WQmu%=_7p9oFk9?e`rDQvN=?q(ut{ltYf=DW ze)Q0&g_Q*rtURcMgEbY*6Tq^P?7pLmz)=B5%_;`8z|#mkg@9JIz{?kTb~G#r$jI)j zC>T}%Ce`7ad-sN+Jv#y1;@KG@1Yq1kRtZ?Oh*bvlj2t8l0yQ#9NShc^*UZR(TDEbm z^#yDmfV*w542B>)OdjrU^~qGgRXtm7)T*<96wbbne-FRjX0>WqHKPJ3sm(%37jXWG z09*Ww&DDJJwFl8DKuS**Qs%STI>bnWl5*&4L?`wLmkUJ%KjQ8_rJOBb3IawEbLtrK zC*OuS3%AuVkvdgOVGf$GMNH9%HP__mqfl|%@Pal0bpqsN4I=`iCz7L&=4u1#i~{Ne zki69fBM;i%7j+sKL%>NU$7j#gXH9@B^9AhbCDTF%xgA;y7aUAJV4KjdejfN00~X!9tbxjNqCMd-W`3{{7)TimM9XO5_5@UnK;7JoFy=j< z-Mq|zTLeZMEZ(jpYwoKMTL!Ez4N_#5f?mOgN(3J&t(UQu5AYic{;Rf0SCaY7F44J)yf3b$_3RbKsDP3sAkK~5`L=P zYlb5O3{rR|0ItIdk}>vC^z4yhsT>mcPltfnXlqX#<3HPGg1L_`&dVpW4kh@ zpir~}b8qSjn7azwI=@x>HE5f=0)(eqCL&kC?A(`0PSUVzB9a0-M=nI5?0F*g0{(U9 zC+R*F{z`S}z7GVgo5+&@ZSLz(F|+~^aVK)k;%vKy0O{Cl44@Qu0so|{#2{v37X(0^ z&EVxOeVoAPoa7imPTM573qnbaYU`>=Ol?()i6Yx7WlP?_QfNE|h+jm5BDU}8cEJ@3 zKx{nUWd}ee#rgYH_y=7T*|Y(_0{|b&=}QbgT4The-OW0pMQ?Cuz;mft&<(fluct2J zz&<1XdUwfDpC>wZW{1-M_n}~ieJ^$Hv@8C8_pTmpk<8ZT2>AvOUYRpnE)~1LYjb6S z;d0rmNtS-o{Vh6f+=lc&P8t(-(*#LdSBy-oqQ^LYZi9}96)bzA!|pBDnKeK8gpk8%*9V5`l+_Oa~IhI;(TlXK*FBEV{*+bxn&LuwP63rB?57%?jM2GK1=$TF@w&@P{bWYme-115sM$&UsI zo;c8~yG*ut6^w5Sf}8QL4phbq^$A(p?t;6#w7EQ?*-#tirK@U3+Gt4}OM6OFT4&l( z_4G(uYnm@x)w;9gWny_^BqNvQ)ugrh&x}4jnsS{=Ju;O#HJzG{q#Pew)jpOs*scs+ z9$Gh)-8Ph^J^rL4qhtx>&*6GU{Nm-j4aY;tyo{1|wQRH;Of?_6DNPNWPW6vwka~`(6wWLO0^B#WKzL1siWf{IYDVN6-Jvb zaU@fRK)J!E2j^Q?HRWl&^-9;}t`&W0;y~KxU-z}eZ~NL-Ozq3QwuO@U-B->h&qI=c z^7c3P^{$u>-YC7^Fu(hA=dZ>9b}Cj%n^xS-x6RErN{P%t=-acZ_ciq--+tzUH$cn8ip6G+C zSzFl)7q1tkItSPG4z5-VtyzZ>M>b5J#KB)0ZS(b?KbfS`I`h@NsoIWJd*_<2GpR^h zJ>tD_s;Xnf+L@GZz?JdwRedR37N=bGYlen(L+i4k^?J)o%|Gb6W9Zv3x>D|jHDlwt zv2EGdmbR2E3Rf^p;85@08r~5XoDO>VXEytuE_8 zo#ZtO1?ksZR_Q=9daX`IG7S=vZY~;BN_;Yo+kf^U*$g;1y0_R z38!Q~;Q&Fx2SNDgGbWx#Owdy;y;M8!0v#`aH1mL7qC#9DQ3yT-=;1Az=>0U<>YV0y zqMXn-0bJRh#j3=_2cR^12@((maDx*i<|+4l6vA%Ri3a){jmJ*Z zE65mOOnCl;Q};d6-uDR+nh)TgobbnmS%X_I_^3tZ3b?#B_mztv2l&1PS9<4x$woOf zw8rOUg(0scdF|OyG&UxHfJp2jub77KX0zNy;VTLEZD??tjW4Nt0*5`q2ewuZv2q@I z>re1-MHGJGq>XCBS5A6|1gjGl<)t&>3o%Zxz2L2aM+iHlVEfJEsBnkoQwfIPezQ45 zVX)uB%2Um|g{dK5wce0-}%Xfl(c$mu64hmSlZ|ciwvA5aID1*u&OwRv?ka*l;>ljK%TRB-1y*=CBUDAWPf zo?CkD(Hm6O5=0A_;&A9}Z2Dp^pEoy?O$!S-yKdy7b%f|6Tsc0V6D`7(2UiQP5!)Jr zef|I>R$^`AUf{Tol?ILm?=x8$TH~7e;*C?1)7zU}7bt?Siq+a$A)PHg53zLNu zt*7u`Pc`e0f`m`P9Rvn1WoNkYE{L92J5p@%(GLrHl_@>4y+t1BT%$*H)a>ubXa$Y;6q^$RlKqqudS0f2Z I_&1LK0Xo8=oB#j- delta 4928 zcmai2Yj9h~b>4e%@qX~)LE=e*6rUhPQUV#iBuk>L2S`yA1;bQn(jp-Oq+pUDxBwkX zmJPLO6FTX*lvjy4NgB&ZnvrWy#xBywWZcAx;#i97nzUpIzD+CHnMo?oG^J%vYEL{) z&$(BiC|lFsA0N&=dv^DnJ!ik&i}(M)ow>|e8Z2f5f#=VyV+q3pFIwIfxUSrP3OvU) zY1-MmW{xwk-)bIU2eb#dhuE{)1I($bR$bQl*dEmTV3U*c3S}7}vL2-gy zE3*Gp_VMhMqQ;e?#*0PG%erP-1ubkvD9QEdD|pV&lE!APf_=x>##QCsGPd)^HZY(n zgdlTTD!`=Ea+jt86q1WQV{x%lmTE4T`?2K%f$L;Hv=4GSnA-6*u8o~^ILd<9u^Zto zgiY8@9jA1b26WWV&N+6$Z2IC>ZfkB^@uwU=&9QCH_rxZW5(asZ?Cu~Wr5+^gfanO0 zSfRK4460i(M0g6nDb1j!R#A~j@YYB6!YdndIsKY~ji2o8B1BKh$e>V30^BjazA0c} z-}Kaa?uV}Q2!NzYXQ-r!rc&`_ET9D!UEcBy`tdl;jK-pwD2<|J3?Quqs415pSn3~m zw`y^e^tTAR?ycqW%=%7vc1-Fd|xSg{A5uRN1@uIZw^b7AMzBKzZ+ z#i1+3)hor-r(>6k8{StFPt~f1i0);bOU_O1FG@{Xu9iJnwUL`;Kdma+zqXKc4CcdS zgda&5pG_v|!zgOTXQ$_9l9E>WP0Q{|Dmwc`ItlG*9l%Y;%fa)i>Q-(tH&ori8BNdv zi)Z>^K+WFRRhj#7&EN1f5kp~NW6;w_!X**CiG-a&Rm!-J54%#v2tVLu&xm&RM7=?6 zil}bAdCd{cfE`Bt%ro@j(Ch5)>#NO{dA^Cv^Sh0N%>9mK_qer1DGT)Rvlp$NI+5Hb z-ZXGQA&4VbcJeb)S&$+{5s~d}@U~b}V%QzAhCN|#L?8BzRIgcB3l$J9jhMn^5p%Q9 z%tzD%wKxe|-66!viJz2`U0mOHx2Q@}h3q^>Zb zhQ#j1l}g!0N(#Dx23(;n{)o*)f_%XtKMA{=NVt5&T(FDS3T@5&JYPz}6)D?x+j?h9 z6jg#tkb-k-&HvtKCto?$}E3+7@QKQLX+r#Xoj;(vtbMtX} zR8pr$aZ9S>_(M-%`=28G83JxZrKqvUYVc8ZWC5YR^PO$lZ$WGNd7e!S`g5&aZJe!5 z?}lww8aT6m#kXZ)-*1b2tb1!eJGs@#p51BLk#E22AG0rJFSUm*+Iv>oLocXK?fkZV z>EP+{i|+c%js{RI>|D0@z!+BNl8_fm@{a1$$McR2-@oUK@Vy@P-dz=SS)Q2PPwxB! z7%9KIIZkhV+a!K+qn{(rd$^A!d(TH7y4aR&eOX-5v(3&>CHa}p8EPOuch-km$Sb@T zv8>zOBamOTTSHyKt2zhBulmYD+l5!VTp*vXf?=pb?f?(_w zRTqkCdX1_JU*J$~6MF6BmqsK**eXuOS@q5a&XPN@GtC)+5ELrXAQg>u zgtaO4As5g*7uI$B2_Z3!%td<8o=Iu;89{H5^=esfOsT`Bl*UWg+-|qlJg5f7vapl8 zJ6m9!h&rs5ZK2?S&x%r7;4v|!4O=5xu!G$OXpd+B9YJkM$2RYEayr($Cm`x0`ZYEL zvh}f4ug7dyM`C}ABvx!p86tXMFVoEyC$N`^z0>Q+4!zXMLul|6_IW?Q*R32&z$vDP z5LVSd>miadqr4GzOyPm^(1!y7i50KcN6a0kNa5Injqus_lzDHNGQ*JjQkcWxfy6OL zryQg!s7_hrAT4aNxAODxsEZ&T6?@s~U<2EA;++EP8v5k_4UCs(cFQw+?bCB*?0VmGm0yM7 zB)&aH7g12PY>Lq*K}sw`s_8dCqEE52`@YJ3hrMxcHT&=VUREA4uutzbv+4m4$PRXJ zz|9})V}I;+vXcXwbi2d*_mv+D8raG}kUPk-kz&?=K+kFpl(7G5)3e7~Jh@K~JjZ8E z>v;UumX%re_I-QD4XZG(q^Hs3WPCJACvWXtNgqu{r;lb5<7xT>4B#sm8CYiGqj0TA zO9o_T_TYUsjl#z4mHW1qO@dKcVd4S-ul)iO2L{`;4IG$ULnaPoU9}6Qyl7icUo$$E zgiA(W-Wyox%R8Kp@5-A?mZ~qA{dsS-tXbH#D%{HfX$xv|G0^Q(mXns{iU*btJ-Gb9 z)biByvVZ21n9Ms%pUR%hUUAmF>8#6F1{eEQO+@r9WiJ&q<$Xs_>aY2BE$U&6GV7{= z)Y!a>##O@mY}agMi)E{XD-Y&_?Jqc2iK^4QYBhSqh2B*Ud*Yxg3s<##W6PDs?Qb@2 zzv$Vq(zyNf_|n|dZA)z+E^3!OJKi3=|6*`>W$^y<=^wSer23~l7dv_`xAcNyB{;mO zU-mSuh|O>m_{$fKd2iKnb>|gt*Rr>3@t(ZXv$X9kXZ@Ms%g&B`P2fz^TQ!~EG{Rlb zw*@-1Y+Lqd;Wc(<7{4NZH(aVwtB-RxEKD46R2tOo8=j!r{oxUAjstl@ zpA5goMtSw_3A}uB=TRT%4`zYNcx+)RGR*ZuurDEJ(y0q1h8(|jv2cLddhd9);s2Jmlz2~5BIC^Z1rrp)O0bOOUvNuVd6t%%bp4{v6f6dp!rY3S3)AvyNXM~?1 zC?2faXYAC&Rl{T0q_|QfR|zFBmQo~gWxR$4vQsLQWHx|HE1?|PokrG)j!gZ{Z|y_|80QPQnUzPF%J1lAQP9FN8tl3Jc?yPF-`Ik zQuKm)u#db;aBk`WkPP7M;n~^gcr;1fs1OoK;Jy3MbQPfrp@LYTNw;-oHa0&U-$v`$ ZyN^6%?$&Vb_lb-lQ!ajdffLaBe*w9fq*4F? diff --git a/APP/openvpn_api_v3.py b/APP/openvpn_api_v3.py index 2eb282a..86f69f7 100644 --- a/APP/openvpn_api_v3.py +++ b/APP/openvpn_api_v3.py @@ -243,11 +243,51 @@ class OpenVPNAPI: pass try: - # 4. Формирование запроса - # В агрегированных таблицах нет полей rate_mbps, возвращаем 0 + # 4. Request Formation is_aggregated = target_table != 'usage_history' - if is_aggregated: + # High resolution handling for 1h and 3h ranges + is_high_res = False + interval = 0 + points_count = 0 + + if target_table == 'usage_history': + if duration_hours <= 1.1: + is_high_res = True + interval = 30 + points_count = 120 + elif duration_hours <= 3.1: + is_high_res = True + interval = 60 + points_count = 180 + elif duration_hours <= 6.1: + is_high_res = True + interval = 120 # 2 minutes + points_count = 180 + elif duration_hours <= 12.1: + is_high_res = True + interval = 300 # 5 minutes + points_count = 144 + elif duration_hours <= 24.1: + is_high_res = True + interval = 900 # 15 minutes + points_count = 96 + + if is_high_res: + query = f''' + SELECT + datetime((strftime('%s', uh.timestamp) / {interval}) * {interval}, 'unixepoch') as timestamp, + SUM(uh.bytes_received) as bytes_received, + SUM(uh.bytes_sent) as bytes_sent, + MAX(uh.bytes_received_rate_mbps) as bytes_received_rate_mbps, + MAX(uh.bytes_sent_rate_mbps) as bytes_sent_rate_mbps + FROM usage_history uh + JOIN clients c ON uh.client_id = c.id + WHERE c.common_name = ? AND uh.timestamp BETWEEN ? AND ? + GROUP BY datetime((strftime('%s', uh.timestamp) / {interval}) * {interval}, 'unixepoch') + ORDER BY timestamp ASC + ''' + elif is_aggregated: query = f''' SELECT t.timestamp, @@ -280,13 +320,44 @@ class OpenVPNAPI: cursor.execute(query, (common_name, s_str, e_str)) columns = [column[0] for column in cursor.description] - data = [dict(zip(columns, row)) for row in cursor.fetchall()] + db_data_list = [dict(zip(columns, row)) for row in cursor.fetchall()] + final_data = db_data_list + + if is_high_res: + # Zero-filling + final_data = [] + db_data_map = {row['timestamp']: row for row in db_data_list} + + # Align to nearest interval + ts_end = end_date.timestamp() + ts_aligned = ts_end - (ts_end % interval) + aligned_end = datetime.utcfromtimestamp(ts_aligned) + + # Generate points + start_generated = aligned_end - timedelta(seconds=points_count * interval) + + current = start_generated + for _ in range(points_count): + current += timedelta(seconds=interval) + ts_str = current.strftime('%Y-%m-%d %H:%M:%S') + + if ts_str in db_data_map: + final_data.append(db_data_map[ts_str]) + else: + final_data.append({ + 'timestamp': ts_str, + 'bytes_received': 0, + 'bytes_sent': 0, + 'bytes_received_rate_mbps': 0, + 'bytes_sent_rate_mbps': 0 + }) + return { - 'data': data, + 'data': final_data, 'meta': { - 'resolution_used': target_table, - 'record_count': len(data), + 'resolution_used': target_table + ('_hires' if is_high_res else ''), + 'record_count': len(final_data), 'start': s_str, 'end': e_str } @@ -342,64 +413,105 @@ class OpenVPNAPI: 'traffic_distribution': {'rx': 0, 'tx': 0} } - # 1. Определяем таблицу и временную метку - target_table = 'usage_history' + # 1. Configuration hours = 24 + interval_seconds = 900 # 15 min default + target_table = 'usage_history' if range_arg == '7d': - target_table = 'stats_hourly' - hours = 168 # 7 * 24 + hours = 168 + interval_seconds = 6300 # 105 min -> 96 points + target_table = 'stats_hourly' elif range_arg == '30d': - target_table = 'stats_6h' # или stats_daily - hours = 720 # 30 * 24 - + hours = 720 + interval_seconds = 27000 # 450 min -> 96 points + target_table = 'stats_hourly' # Fallback to hourly/raw as needed + + # Fallback logic for table existence try: - # Проверка наличия таблицы - try: - cursor.execute(f"SELECT name FROM sqlite_master WHERE type='table' AND name='{target_table}'") - if not cursor.fetchone(): - target_table = 'usage_history' - except: - pass + cursor.execute(f"SELECT name FROM sqlite_master WHERE type='table' AND name='{target_table}'") + if not cursor.fetchone(): + target_table = 'usage_history' # Fallback to raw if aggregated missing + except: + target_table = 'usage_history' - # 2. Глобальная история (График) - # Для агрегированных таблиц поля rate могут отсутствовать, заменяем нулями + try: + # 2. Global History (Chart) if target_table == 'usage_history': rate_cols = "SUM(bytes_received_rate_mbps) as total_rx_rate, SUM(bytes_sent_rate_mbps) as total_tx_rate," else: rate_cols = "0 as total_rx_rate, 0 as total_tx_rate," + # Aggregation Query + # Group by interval_seconds query_hist = f''' SELECT - timestamp, - SUM(bytes_received) as total_rx, - SUM(bytes_sent) as total_tx, - {rate_cols} - COUNT(DISTINCT client_id) as active_count - FROM {target_table} - WHERE timestamp >= datetime('now', '-{hours} hours') - GROUP BY timestamp + datetime((strftime('%s', timestamp) / {interval_seconds}) * {interval_seconds}, 'unixepoch') as timestamp, + SUM(total_rx) as total_rx, + SUM(total_tx) as total_tx, + MAX(total_rx_rate) as total_rx_rate, + MAX(total_tx_rate) as total_tx_rate, + MAX(active_count) as active_count + FROM ( + SELECT + timestamp, + SUM(bytes_received) as total_rx, + SUM(bytes_sent) as total_tx, + {rate_cols} + COUNT(DISTINCT client_id) as active_count + FROM {target_table} + WHERE timestamp >= datetime('now', '-{hours} hours') + GROUP BY timestamp + ) sub + GROUP BY datetime((strftime('%s', timestamp) / {interval_seconds}) * {interval_seconds}, 'unixepoch') ORDER BY timestamp ASC ''' cursor.execute(query_hist) rows = cursor.fetchall() - if rows: - columns = [col[0] for col in cursor.description] - analytics['global_history_24h'] = [dict(zip(columns, row)) for row in rows] - - # Максимум клиентов - max_clients = 0 - for row in analytics['global_history_24h']: - if row['active_count'] > max_clients: - max_clients = row['active_count'] - analytics['max_concurrent_24h'] = max_clients - - # 3. Топ-3 самых активных клиентов (за выбранный период) - # Внимание: для топа всегда берем данные, но запрос может быть тяжелым на usage_history за месяц. - # Лучше использовать агрегаты, если период большой. + columns = [col[0] for col in cursor.description] + db_data = {row[0]: dict(zip(columns, row)) for row in rows} - # Используем ту же таблицу, что и для истории, чтобы согласовать данные + # Post-processing: Zero Fill + analytics['global_history_24h'] = [] + + now = datetime.utcnow() + # Round down to nearest interval + ts_now = now.timestamp() + ts_aligned = ts_now - (ts_now % interval_seconds) + now_aligned = datetime.utcfromtimestamp(ts_aligned) + + # We want exactly 96 points ending at now_aligned + # Start time = now_aligned - (96 * interval) + start_time = now_aligned - timedelta(seconds=96 * interval_seconds) + + current = start_time + # Generate exactly 96 points + for _ in range(96): + current += timedelta(seconds=interval_seconds) + ts_str = current.strftime('%Y-%m-%d %H:%M:%S') + + if ts_str in db_data: + analytics['global_history_24h'].append(db_data[ts_str]) + else: + analytics['global_history_24h'].append({ + 'timestamp': ts_str, + 'total_rx': 0, + 'total_tx': 0, + 'total_rx_rate': 0, + 'total_tx_rate': 0, + 'active_count': 0 + }) + + # Max Clients metric + max_clients = 0 + for row in analytics['global_history_24h']: + if row.get('active_count', 0) > max_clients: + max_clients = row['active_count'] + analytics['max_concurrent_24h'] = max_clients + + # 3. Top Clients & 4. Traffic Distribution (Keep existing logic) + # Use same target table query_top = f''' SELECT c.common_name, @@ -417,7 +529,6 @@ class OpenVPNAPI: top_cols = [col[0] for col in cursor.description] analytics['top_clients_24h'] = [dict(zip(top_cols, row)) for row in cursor.fetchall()] - # 4. Распределение трафика query_dist = f''' SELECT SUM(bytes_received) as rx, diff --git a/DOCS/api_v3_endpoints.md b/DOCS/api_v3_endpoints.md index 02580ca..4d214f1 100644 --- a/DOCS/api_v3_endpoints.md +++ b/DOCS/api_v3_endpoints.md @@ -1,90 +1,142 @@ -# OpenVPN Monitor API v2 Documentation +# OpenVPN Monitor API v3 Documentation -Этот API предоставляет доступ к данным мониторинга OpenVPN, включая статус клиентов в реальном времени и исторические данные, хранящиеся в Time Series Database (TSDB). +This API provides access to OpenVPN monitoring data, including real-time client status and historical data stored in a Time Series Database (TSDB). It features optimized aggregation for fast visualization. **Base URL:** `http://:5001/api/v1` --- -## 1. Статистика по клиенту (Детальная + История) +## 1. Global Analytics (Dashboard) -Основной эндпоинт для построения графиков и отчетов. Поддерживает динамическую агрегацию данных (умный выбор детализации). +Provides aggregated trend data for the entire server. optimized for visualization with exactly **96 data points** regardless of the time range. -### `GET /stats/` +### `GET /analytics` -#### Параметры запроса (Query Parameters) - -| Параметр | Тип | По умолчанию | Описание | +#### Query Parameters +| Parameter | Type | Default | Description | | :--- | :--- | :--- | :--- | -| `range` | string | `24h` | Период выборки. Поддерживаются форматы: `24h` (часы), `7d` (дни), `30d`, `1y` (годы). | -| `resolution` | string | `auto` | Принудительная детализация данных.
**Значения:**
`auto` — автоматический выбор (см. логику ниже)
`raw` — сырые данные (каждые 10-30 сек)
`5min` — 5 минут
`hourly` — 1 час
`6h` — 6 часов
`daily` — 1 день | +| `range` | string | `24h` | Time range. Supported: `24h`, `7d`, `30d`. | -#### Логика `resolution=auto` -API автоматически выбирает таблицу источника данных в зависимости от длительности диапазона: -* **≤ 24 часов:** `usage_history` (Сырые данные) -* **≤ 7 дней:** `stats_hourly` (Агрегация по часам) -* **≤ 3 месяцев:** `stats_6h` (Агрегация по 6 часов) -* **> 3 месяцев:** `stats_daily` (Агрегация по дням) - -#### Пример запроса - -```http -GET /api/v1/stats/user-alice?range=7d - -``` - -#### Пример ответа +#### Behavior +* **24h**: Returns 15-minute intervals. +* **7d**: Returns 105-minute intervals (1h 45m). +* **30d**: Returns 450-minute intervals (7h 30m). +* **Zero-Filling**: Missing data periods are automatically filled with zeros to ensure graph continuity. +#### Example Response ```json { "success": true, - "timestamp": "2026-01-08 14:30:00", + "timestamp": "2026-01-09 12:00:00", + "range": "24h", "data": { - "common_name": "user-alice", - "status": "Active", - "real_address": "192.168.1.50:54321", - "last_activity": "N/A", - "current_rates": { - "recv_mbps": 1.5, - "sent_mbps": 0.2 - }, - "totals": { - "received_mb": 500.25, - "sent_mb": 120.10 - }, - "meta": { - "resolution_used": "stats_hourly", - "start": "2026-01-01 14:30:00", - "end": "2026-01-08 14:30:00", - "record_count": 168 - }, - "history": [ + "global_history_24h": [ { - "timestamp": "2026-01-01 15:00:00", - "bytes_received": 1048576, - "bytes_sent": 524288, - "bytes_received_rate_mbps": 0, - "bytes_sent_rate_mbps": 0 + "timestamp": "2026-01-09 11:45:00", + "total_rx": 102400, + "total_tx": 51200, + "active_count": 5 }, ... - ] + ], + "max_concurrent_24h": 12, + "top_clients_24h": [ ... ], + "traffic_distribution": { "rx": 1000, "tx": 500 } } } - ``` -> **Примечание:** Поля `*_rate_mbps` в массиве `history` возвращают `0` для агрегированных данных (hourly, daily), так как агрегация хранит только суммарный объем трафика. - --- -## 2. Текущая статистика (Все клиенты) +## 2. Client Statistics (Detail + History) -Возвращает мгновенный снимок состояния всех известных клиентов. +Main endpoint for individual client reports. Supports **Dynamic Aggregation** to optimize payload size (~98% reduction for 24h view). + +### `GET /stats/` + +#### Query Parameters +| Parameter | Type | Default | Description | +| :--- | :--- | :--- | :--- | +| `range` | string | `24h` | Time range. Formats: `1h`, `3h`, `6h`, `12h`, `24h`, `7d`, `30d`. | +| `resolution` | string | `auto` | Force resolution (optional): `raw`, `5min`, `hourly`, `auto`. | + +#### Dynamic Aggregation Logic (`resolution=auto`) +The API automatically selects the aggregation interval based on the requested range to balance detail and performance: + +| Range | Resolution | Points | Source Table | +| :--- | :--- | :--- | :--- | +| **1h** | **30 sec** | 120 | `usage_history` (Raw) | +| **3h** | **1 min** | 180 | `usage_history` (Raw) | +| **6h** | **2 min** | 180 | `usage_history` (Raw) | +| **12h** | **5 min** | 144 | `usage_history` (Raw) | +| **24h** | **15 min** | 96 | `usage_history` (Raw) | +| **7d** | **1 Hour** | 168 | `stats_hourly` | +| **30d** | **6 Hours** | 120 | `stats_6h` | + +*All short-term ranges (≤24h) include automatic **Zero-Filling**.* + +#### Examples: Long-Term Aggregated Data + +To explicitly request data from long-term storage tables (skipping raw data), use the `resolution` parameter or specific ranges. + +**1. Last 7 Days (Hourly Resolution)** +Uses `stats_hourly` table. Reduced precision for weekly trends. +```http +GET /api/v1/stats/user-alice?range=7d +``` +*or explicit resolution:* +```http +GET /api/v1/stats/user-alice?range=7d&resolution=hourly +``` + +**2. Last 30 Days (6-Hour Resolution)** +Uses `stats_6h` table. Ideal for monthly volume analysis. +```http +GET /api/v1/stats/user-alice?range=30d +``` + +**3. Last 1 Year (Daily Resolution)** +Uses `stats_daily` table. Extremely lightweight for annual reporting. +```http +GET /api/v1/stats/user-alice?range=1y&resolution=daily +``` + +#### Example Response +```json +{ + "success": true, + "data": { + "common_name": "user-alice", + "status": "Active", + "current_rates": { "recv_mbps": 1.5, "sent_mbps": 0.2 }, + "totals": { "received_mb": 500.25, "sent_mb": 120.10 }, + "history": [ + { + "timestamp": "2026-01-09 11:30:00", + "bytes_received": 5000, + "bytes_sent": 2000, + "bytes_received_rate_mbps": 0.5, + "bytes_sent_rate_mbps": 0.1 + }, + ... + ], + "meta": { + "resolution_used": "usage_history_hires", + "record_count": 120 + } + } +} +``` + +--- + +## 3. Current Statistics (All Clients) + +Returns a snapshot of all known clients. ### `GET /stats` -#### Пример ответа - +#### Example Response ```json { "success": true, @@ -93,110 +145,63 @@ GET /api/v1/stats/user-alice?range=7d { "common_name": "user-alice", "status": "Active", - "real_address": "192.168.1.50:54321", "current_recv_rate_mbps": 1.5, - "current_sent_rate_mbps": 0.2, - "total_received_mb": 500.25, - "total_sent_mb": 120.10, - "last_activity": "N/A" + "total_received_mb": 500.2 }, - { - "common_name": "user-bob", - "status": "Disconnected", - "real_address": null, - "current_recv_rate_mbps": 0, - "current_sent_rate_mbps": 0, - "total_received_mb": 1500.00, - "total_sent_mb": 300.00, - "last_activity": "2026-01-08 10:00:00" - } + ... ] } - ``` --- -## 3. Системная статистика +## 4. System Statistics -Сводная информация по всему серверу OpenVPN. +Aggregated metrics for the Server. ### `GET /stats/system` -#### Пример ответа - +#### Example Response ```json { "success": true, "data": { "total_clients": 15, "active_clients": 3, - "total_bytes_received": 10737418240, - "total_bytes_sent": 5368709120, "total_received_gb": 10.0, "total_sent_gb": 5.0 } } - ``` --- -## 4. Сертификаты +## 5. Certificates -Информация о сроках действия SSL сертификатов пользователей. +SSL Certificate expiration tracking. ### `GET /certificates` -#### Пример ответа - +#### Example Response ```json { "success": true, "data": [ { "file": "user-alice.crt", - "common_name": "user-alice", "days_remaining": "360 days", - "is_expired": false, - "not_after": "Jan 8 12:00:00 2027 GMT" + "is_expired": false } ] } - ``` --- -## 5. Вспомогательные методы - -### Список клиентов (Упрощенный) - -Используется для заполнения выпадающих списков в интерфейсе. +## 6. Utility Methods ### `GET /clients` - -```json -{ - "success": true, - "data": [ - {"common_name": "user-alice", "status": "Active"}, - {"common_name": "user-bob", "status": "Disconnected"} - ] -} - -``` - -### Проверка здоровья (Health Check) - -Проверяет доступность базы данных. +Simple list of clients (Common Name + Status) for UI dropdowns. ### `GET /health` - -```json -{ - "success": true, - "status": "healthy" -} - -``` \ No newline at end of file +Database connectivity check. Returns `{"status": "healthy"}`. \ No newline at end of file diff --git a/DOCS/data_gathering_report.md b/DOCS/data_gathering_report.md new file mode 100644 index 0000000..8a402c2 --- /dev/null +++ b/DOCS/data_gathering_report.md @@ -0,0 +1,45 @@ +# OpenVPN Data Gatherer Analysis + +This report details the internal mechanics of `openvpn_gatherer_v3.py`, responsible for collecting, processing, and storing OpenVPN usage metrics. + +## 1. Raw Data Collection (`run_monitoring_cycle`) + +The gatherer runs a continuous loop (default interval: **10 seconds**). + +### A. Log Parsing +- **Source**: `/var/log/openvpn/openvpn-status.log` (Status File v2, CSV format). +- **Target Fields**: `Common Name`, `Real Address`, `Bytes Sent`, `Bytes Received`. +- **Filtering**: Only lines starting with `CLIENT_LIST`. + +### B. Delta Calculation (`update_client_status_and_bytes`) +The log provides *lifetime* counters for a session. The gatherer calculates the traffic *delta* (increment) since the last check. +- **Logic**: `Increment = Current Value - Last Saved Value`. +- **Reset Detection**: If `Current Value < Last Saved Value`, it assumes a session/server restart and counts the entire `Current Value` as the increment. + +### C. Rate Calculation +- **Speed**: Calculated as `Increment * 8 / (Interval * 1_000_000)` to get **Mbps**. +- **Storage**: Raw samples (10s resolution) including speed and volume are stored in the `usage_history` table. + +## 2. Data Aggregation (TSDB) + +To support long-term statistics without storing billions of rows, the `TimeSeriesAggregator` performs real-time rollups into 5 aggregated tables using an `UPSERT` strategy (Insert or update sum). + +| Table | Resolution | Timestamp Alignment | Retention (Default) | +|-------|------------|---------------------|---------------------| +| `usage_history` | **10 sec** | Exact time | 7 Days | +| `stats_5min` | **5 min** | 00:00, 00:05... | 14 Days | +| `stats_15min` | **15 min** | 00:00, 00:15... | 28 Days | +| `stats_hourly` | **1 Hour** | XX:00:00 | 90 Days | +| `stats_6h` | **6 Hours** | 00:00, 06:00, 12:00... | 180 Days | +| `stats_daily` | **1 Day** | 00:00:00 | 365 Days | + +**Logic**: Every 10s cycle, the calculated `Increment` is added to the sum of *all* relevant overlapping buckets. A single 5MB download contributes immediately to the current 5min, 15min, Hourly, 6h, and Daily counters simultaneously. + +## 3. Data Retention + +A cleanup job runs once every 24 hours (on day change). +- It executes `DELETE FROM table WHERE timestamp < cutoff_date`. +- Thresholds are configurable in `config.ini` under `[retention]`. + +## Summary +The system employs a "Write-Optimized" approach. Instead of calculating heavy aggregates on-read (which would be slow), it pre-calculates them on-write. This ensures instant dashboard loading times even with years of historical data. diff --git a/UI/client/src/App.vue b/UI/client/src/App.vue index ef21163..386dab4 100644 --- a/UI/client/src/App.vue +++ b/UI/client/src/App.vue @@ -30,11 +30,14 @@ + - + @@ -46,6 +49,7 @@ import { useAppConfig } from './composables/useAppConfig'; const { loadConfig, isLoaded } = useAppConfig(); const timezoneAbbr = ref(new Date().toLocaleTimeString('en-us',{timeZoneName:'short'}).split(' ')[2] || 'UTC'); const isDark = ref(false); +const refreshKey = ref(0); const toggleTheme = () => { isDark.value = !isDark.value; @@ -54,6 +58,10 @@ const toggleTheme = () => { localStorage.setItem('theme', theme); }; +const refreshPage = () => { + refreshKey.value++; +}; + onMounted(async () => { await loadConfig(); diff --git a/UI/client/src/components/HistoryModal.vue b/UI/client/src/components/HistoryModal.vue index ef21508..859ca9d 100644 --- a/UI/client/src/components/HistoryModal.vue +++ b/UI/client/src/components/HistoryModal.vue @@ -18,20 +18,9 @@ @change="loadHistory"> - - - - - - - - - - - + - - + diff --git a/UI/client/src/views/Analytics.vue b/UI/client/src/views/Analytics.vue index a0ae004..1b33e05 100644 --- a/UI/client/src/views/Analytics.vue +++ b/UI/client/src/views/Analytics.vue @@ -178,7 +178,7 @@ const expiringCertsList = ref([]); let cachedHistory = null; // Helpers -const MAX_CHART_POINTS = 48; +const MAX_CHART_POINTS = 96; const loadAnalytics = async () => { loading.analytics = true;