From 626d4a5a565ebf65b20ec70235b9e1cc7d5f5228 Mon Sep 17 00:00:00 2001 From: robviren Date: Sat, 30 May 2026 18:25:42 -0500 Subject: [PATCH] Better live handling --- README.md | 0 __pycache__/infer.cpython-312.pyc | Bin 0 -> 16357 bytes bench.py | 260 ++++++++++++++++++++++++++++++ live_onnx.py | 176 ++++++++++---------- pyproject.toml | 3 + quantize.py | 182 +++++++++++++++++---- uv.lock | 133 +++++++++++++++ 7 files changed, 639 insertions(+), 115 deletions(-) delete mode 100644 README.md create mode 100644 __pycache__/infer.cpython-312.pyc create mode 100644 bench.py diff --git a/README.md b/README.md deleted file mode 100644 index e69de29..0000000 diff --git a/__pycache__/infer.cpython-312.pyc b/__pycache__/infer.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..05539bfe649016f1c36b38df072edc7f0dc26d54 GIT binary patch literal 16357 zcmb_@Yj9K9nc%(B)mye~`H2iz#$YUiZH(P+9?ioKUhc+pV?%?1R)p`hEn!RMUKy}j z&bYIgjTFdQ-kyo&f(&9eRhTqenA%LWdwxvlOsbkxQnk9s87ntEwaH|vX7i7RRLy38 z?C$rSt9vCI(XcanR66%OzVo=}eCKUgGJWT^;miaa{U}1F-TUkBC1sduI z#Tvd%vBpm{@)!e!q|5|mX0^=56|t5}^%ND;@GBaMqkWr{sb!y^WRe4R&hFcqEmu;k zl`XhmsPsuwFM+bX7F5wEwA}AY`zdDa_e)+mTIp+zR`yy@Fje=+v+uKs}wlsTjSfM!834 zRB=_T6DK;b5qh+7Wn7iAmIKvL;$&+SSirApUzgHT_6bl{c*E{&iIUaxB1?HQ<^Y)d7Tg&L@Xd7n_qvrn#0nZ>OOHG6J^)Fw%t>smzPCbg~UxE3Ab4mOE7F3@n}= zcKc0+ZB$wVY|{@63LH&MY9=+VX3rZy+|4J3hDN=9&N<{Ak) z!tsJP;CD_4UjMK&IKnx(YvTbv$g$27y}f6g!@OsFL})UbyLs;wP9PA1XN+@R_4?Vs zRRA0Jj&o!HkKh~yn5OYbXT892ER1vw>iq$J%roi@d4e#`cIP1H4|05?FbWee2LqQl zzW{|Eo}X+Kf}SAfJl5BLxZmZxI1zO6oQD;hLwsP&=^^uXdi_EW3Y~!=C>iq(HaYtP z{NPCA5YKTiNvCt-ij$x41KbeD<4hVGg}?+4@9On8UG-dne9*%WbJ+J#rU2*a#3+*w zOa#YaaCndc=L?~+k@F7*SPnq|QfQX5O##3E8Vta4t19KJQaL;txab*GTgq8dX*LVO zs0`-uv(7Ot=xOo^ur(|<{95*I<4N|PC06yL)uI0NIj(w@=1 zut&iWn4{Ah+`f7n5Nr@OC z7w!?18L$jWAcyn$*N@OgxpckvXi3KgC&xJ+aq16M46{Xt1P%K4$1_xOjoOjUcE#0-vl$0Z#=26#yaaP4g_lSE&WbOP`8LIZff z;gNK3M!W%u_Dgh7(u@Wq&4^c`hXh1Ul{EY>SMx|CUvt961>+pRq-4nVA`Dr#sTtg2=__p;%wMW%hm1x*%J(05Ft)wN=25iKo0-t=h$j9}T9WcEFW_oN0@k^mV%4Vj!QLt>5!sw4!-%i0ho zD$aeaauCh~%#>zI8`k)+pEZGH?yCoKN|PlkKx(vMTImC7R5Fv7gs1|GuVBppkX1y? zDJIOY`b*tF2zAgugTw@h8X&Hrmxm11@6zy~#Zi*UGkVoCDY$pCJW@KqYQ}TTEBy1G z8$15-KTICl?=nalVMt<-`AN(e=kXIf;gP&aTF*s6(vEqqxpWdU4z!ss!(!S4ZFnf* zJeLGq3WAvYdaN_TQDEII?U(pQ$kxJNxB$^K^`*6N=1BBNtUF;^>|eI-`RLGe$26Zd znr8G-eXJv*zg~#wRTL|Zb;k86V>RR~1vBl@_E^pJ-81{5`>r3D?oL-!#o7P(VWcGz zjP6P@t_2z*ej|*qif!Pl6SncUka*=jYk~kJhkR@!OSW^_1q&=hA?MjW4dUXHki$fP zX(2&Upr>@Of4VS3_OB2K7vvliUK{U`MsRJgzYzLoU6%F}e`P2L!Uy zJg`I9sWGpgM~nf81}xBlc*6%JJrJYe;E1F}+CmEw%?Y?RG9{6Bj0t)!aXczJ+u$#t zBn`V|w$9W?>tpRdwBL2yaV(p+Pj`K3DU4i>ZMnYhR_(39M9s~i1h+7B$DeY%F?|`% zrok9#`PM{wOI_mfLUH1BL>uXd>7vKtbo|h~DQVo4wiHhH$Xf+Ns6jb`&nuy5j^Ze= zHc)730Ly`v)j-TlYF&&}*cSvGh1VP=B3u6L24z5&5~!>g4>AMgRz^Y2Pr22@j`9R9 zIZKD23~E&VuGByoqs%qCxxPHyJxmAj(2(HE!zb@+R0Pnn@XP5f4|ivooZj+qaKiJ5 zbRtVK3P&g059Z;+le#A4^q6OUN)PbHIY{g)U(*R>VYQ58wbWI}c^5|>c-_~JoLw0h zTbUf}m(L>M+=ARgHaiFJd(;0?Onp^DXKqDiS}19;)@AObF5=7=yi%;}O! zco@?q<>XPiF5Wf!cA`Tp*+TMVj(FefIRr#9eJpw`#{F>T-92~qEL)o*bh-knA>V$d zeWCWpnoo5Pb@#R{(f68{4v7sNsoG9)L#J5Lxl-OEmiPQ}=YQMt^F68Z)6knGUGA8> zG*Ro`7g^W=SF5n;+OAh9+>Z$m&;pUUYf30A7Q>{eUZ2W?#aacJgf`q1T3Cn zppi&qfjw(|@LU^M4N8X~!K-IKAF1~tk6aUgFjGDpb4_5i_Zfw)1B80^vtTUPQ2$)R zQbS~HoMAquo--pA_p^N{r*aAy21{rajHP057LL#Z%9oE)0MIHKzBJoPNrm-aKMN3T zGV3;F$k7J=Z9N%-h}LY9mPP^@0T0iwGWz#dpx%0c;1 z$XgRW$>f3BqEJEttoQ4{t7iaazFBFTeG28_0;s9WmMAHl%TxhtfHBom5H9%FpEwY` zjJ=ae@AC8M0U|{czQET&nJk*&3iP{5LFgLsfWnBRK++<^(0O@y$mF~e!I3A(VjH3E zZ()|O1kn3HU*x;6pc|t@7#)B}GJpW?9h(>f9WpRJ$sfjw{Sdhd`6F15viqpSc*i`$ zz&^&gL5c2%ERL_|xnZ!*Nc1HC1~x-P=8-!{njkdc#=Jqk6|w@ZI&zq7ka>+^oDU3g zf)FZS!)h|+9oYDDkgkB!*h*p-<88BE(N-N{($=Dxo@h_(a`bqF{<6qEr=Qi&S!b>B z{zMxxmc@>kHC5EURQi*u&#Hd<-ecy``{M57V$tyx+bPj@YWeh;mDAr4Pk$q2^FU|z z63`O2#M^G~n%@OlWrDpsa%V&=Zi?uUUmcAejh)WnS9!~d%VU?nUm5907Zu04=Z?-E zowY&+6iQF5CdMLHn%$nVRwvqiL@%@~Fn9Kd)<($ZgSfl*&R)^lv}~<@YM|_nr$(yK zK65U5Zlz$OSg^1!x^KLbw%KPw(a^0;w;SdgQnoEg;}*hadqL4Ny4`*-xw_qw+3g++ zuoI)0wz}PyCp@Ea4Sy82oj(iFJ&Lp=$inZ!Qj7>np>X(Z>Y2u(+wj~`uB(#)v3d~x zYEUHe=Xe(K%m1tvv;ZCPE9!|~(Lnr)MizXE118o8v3b%2Mj72PG{mk_4M0SYg@rS~ zz(80)@n0P~3~B}vV<=Om0T8mN0A;GE00c)(tBsN5WAAH%%!qRo2@OjKKJ=$T!x&6v zTvY8yfHGaC3^g>Rj2OD0ttyxRoNVZ#+^SFlWyVZf)w&R-eRS6-IVcn04gv*ir??C#pgwIcPgXRD z_HB?cq|4UDw~1w_a+o1K_x|kr@!;*yd`K*FWy<1PXD^Us<@)%5SXmG0g0y2}JS;l4 zL)w;pS$RoCva((*!G4Ut?LPX&v7a7G7Hv(s4v4mc$$nq5?^1;MvZQ>aWUE-Rb%A+c zyk}hMUM}fC_}p^KJYWAN~&nscRfG*A}B%D`IHc6;yq-jsFginUp^ zHZRhPJ5$znP=bM0ufzP+_ou-BQDv0m%}au4?Ru`$7MY*wDU0lcmdXS3QU{ucp?p2M@%7FfW%zlxbYVZ_iouzC8)GElfMR?3eyn%59)uyDJd$6 zb$x7%ci--r?-7gYB07|4X%t$weAf#EE*6{%&4vF}UVJX`h?gpZ@EW@Mz4gH(K4Bz+vw01jl)-JDA99GmAHYw*2)-{)cXI|kY zrNpg6*1AnuYh@+;n{#HF*M^?6qLtZ|Qc^h^D2%MiGc1~oKdgLX=D zRY?P8tWLZ1GIfzH8%U7Bi?|6M62r2AA!6%r74UxnC43Yk79(V#vdO~&JZTWFc=+Kj zvl2vd+1!Clg`dF)%^f^iXj~SVKI!o)3CN%yV+Cq~l7Vm|!7b5X%K+h0mV~q1NwQ#* zjG!#w$p;Rv1le$5QZJl@Y%4OCu5(DJwCj=zi-*zFYfPqxNfm@ zQHXWNH{W*6yAoYDn-+>i$F?UW+mrkIA_mdkkCu(&(c?e?kIWwtt+im@C@z`XIJ@!Z zTbG?Df4=pzt4qO0?36<`QkL`_)>-SD}~KsVKbQD%@56~ z!o3mgS9+?j?2kICV%?m7*8l4Z?6Q>;FRfdok=;`leYGxvHefai>Z=R^q+q3Jo%^RMyghi&GcY2Q3tW?n_9k*5+$-3Ihvjv5Beopr>sbk-V~rPle+pK@lwX8aLy%)BiL zj|Yb{I<@@J5+Hq=r1bAXUycKjH*lk9O*ysz`%t z8Rx57LY2Nj4}uj_sHlVT)7d*fGY#KzXI%I)g%m^LYqW&q6-GpRxi| zXjRTbZ+yT<*9Mjurdhc2lhpuZ{+ZW5`C6lc{HxF@BvFJ|Nktirfg8KXu`D==rIJiZ zt{@B+)8p6#SDI)LG8sd$32y9+k8^&OxYPLn0wA99a~NgG*oT-Ug0mr`Xat9YC|;qg zEfCWx|1Q8tG}vN?ffx!n#|&wbDU*)OF=Uac63JW#7~vlw0*X{*kLbQcW96;dICrxF zOr4kRT>AUsoskX{t|mU-lb~n!gH%;iI%k?S-P#Pj^qr-SN6Z(N&n=&w_~l^o@(ktelyN3)GXV!uh?2eTkDTIKka$gld>HE@|3F&H7D2a zNVXn*yg9k^M9TIqXhou}Z^bqs+6I=-U06BCiRZYKZ769R`WhwayFii&IY8vkh$65i zYRHlUIPj-3Y+Xl?F`vLjdQ}nmI`ONI${9m-f~X4!j{6)(yn5KO^;t0y4jJ4#K}sQW zeY4E<2?Hmr|36_Bi_w~6g2NIX?=ogih`Dx79}65(23C`{ws8v_05&Z{Jdzxa~l);^42W9l-M(O~6=p?Y+Sh z>>tc!&Gq-{Q?}Nm@!ZceUl>0(J}zE9*}rmfKs-6HeDE9;zlhvOIS8-ohj^7GCa(sm zn4|NVn`OSEQ12-Q_$Kh694%MXZ7#vS#b~}g=Mq|81kTr1z7M000=64Ka~1jEZ8Cv) z5DjqWMde!g_*G7A9(03)g-0t7qT&%!<^VTkvYNd5jx0lT_sqdZlWbm&q8_xb6_?*mY1~9;|hFDJ8M|ytIlji&I1To zO-?qG44X1ua7)Lw0SIf-6U z&}xw5@(GL(w_K&N?`jMSh;Xn2Q-c@{VMJ&gVz7MAXBbmNC?M2s60;~2@Ru;3r1EnM@fGp@}X;qzV2#Y@!+30sj^O zC#cX$^32E;!lXYqqR15_a}ywi0br$Qx7i3Z@yh&_#PIx|Ei#|#AL&}Yyffc()w0o8>vMU#T z;zeJ|ehFwY5b?_S%G>Mb*C#$)d^=fpFnzEqdG@?mcj38~cDU(i4^Z~azb!7C+c3K! zQF^!XPNi7v!r{y38fP05n?-xw!sZ7+okft7Iu#l}F*h|kmFWLb&BDn=&BCdL`a2&i z_Ww=I(#c1frBh4w4?jp%bT3;ErL85gVDv=f$kP(aRxU5Wa$)UPrPS7j943%7zH^^{ zaP8i;CHiOfFC3pcmYd&!GD7luL6sfA9Xp57d5q9&AxoEpXA+(HudwVcMu-CZUt@%q zV@V&fvjcsTJVw}^&;${>P>Xf0XLTCgDf*d_(G@(i>UHJMicGpUpOqVQ!}K$|PIsPu zURa0k`7y~9xG>A@#) zDwzfFlyUALKfyuodT^Y9Pbp*sQ1Wp~$fSOmBH2)s&v@AIOOjBDD(ySv7ins03Kc4P zJ}3d0FC~(LHw0PbgA*to{XS(3_g~^5)*tU<7e?hDjtcYd_TJpQ^el8 z3sVGd5ND*q4t|(?3-B%u$4m6MWc9GD8zw-CB{P`^<^_HV+g`zDdihNy18k}rKKkKh zmkR72|4pn12LXH?C7JM^xI4&Q1E-+CgW{vPGAQZBdHA4#U^ZT9#1EeMhu8=%Gjc5W z`^Yy@d>jWOXCNyJ{s7AeK{SJyn6c={bd%6wJR_2hcz<}Lzmi@?OhPIp-Pk3Vs-(Nh zd%@zg`Y1`px8%zU`}yBML)0gPe*nSgsg|bcuQV2#dG4U-qTf)Kf1oOVPuYJ%t@{mC z{!DM8_dl!D(GAavbhPbXiZ;`AUr`YKOE29^)5qzrDDrrEQA07-NGPRUmo`>Mj4Q@! z(O8`_*1$z*M)#fGk9uQ!7MSVYWv20E`CZG*dbKXObw2_hPzzqscu(T^Qa83xhkJ9G zsZv_!cD*a{4#KshE!LU0qHkT_Ki&PiLVK)jZtv{gRAKc>VVzi5H+>|d&ONcU&9}y{ zE;z*M)>Kv7boaN9W;D$wMQ_gcB`Oy7iZ!S$Ld~(Xr7Y4mvnRS|mIbD;dw%!FftB)R zvAj7|-jcHHgxmTSTe7G|v}}SgO@)zQtUqOPOn3f{F(u8_%ZxKE-!h-w1;(_u=igo^ zO_emI%+1qX1j2!D)X?<6^&?mG3|wOI&UzE2%LR3xRX*DM#kSA4Jr1UJ_fH>AZ>YX~ ze*XMIU1~#%$T+6EBejtaNe9klrX~%xxplMa=65B!@4kKK?ZwhmRr?aNG?*&xobE{% zR7FnCd=UL0er};5x&Dn*LHqP!Xc{>-I~Z^HIFKx>OZbxJmStvVTDfNgi%^x?(6}I^ ztgX|?JR+UsyM{yYwnRzFx@G#%@67P3XxU4dtERisjNv;yAN8y-r6N-r?_cOnGNrg3 zjCH#6J4Zh{8tsdfTt5?=h_i`(;zk_m*D!&@vDVm@m;ie*aT5?};euGdSFGB*)RlA` ie8fEJf7~b@I+N@^n>>3VY5w3>jQi<(G{raw`TxH$>f~_% literal 0 HcmV?d00001 diff --git a/bench.py b/bench.py new file mode 100644 index 0000000..a207ec9 --- /dev/null +++ b/bench.py @@ -0,0 +1,260 @@ +import argparse +import json +import os +import statistics +import time +from pathlib import Path + +import numpy as np +import onnxruntime as ort + +ort.set_default_logger_severity(3) + +NP = { + "tensor(float)": np.float32, "tensor(float16)": np.float16, "tensor(double)": np.float64, + "tensor(int64)": np.int64, "tensor(int32)": np.int32, "tensor(int8)": np.int8, + "tensor(uint8)": np.uint8, "tensor(bool)": np.bool_, +} +TAG = {"tensor(float)": "f32", "tensor(float16)": "f16", "tensor(double)": "f64", + "tensor(int64)": "i64", "tensor(int32)": "i32", "tensor(int8)": "i8", + "tensor(uint8)": "u8", "tensor(bool)": "b"} +GRAPHS = ["ssl", "encode", "decode", "global"] + + +def cpu_info(): + info = {"cpu": platform_cpu(), "logical": os.cpu_count(), "phys": "?", "isa": {}} + try: + txt = Path("/proc/cpuinfo").read_text() + for l in txt.splitlines(): + if l.startswith("model name"): + info["cpu"] = l.split(":", 1)[1].strip(); break + flags = next((l for l in txt.splitlines() if l.startswith("flags")), "") + cc = next((l for l in txt.splitlines() if l.startswith("cpu cores")), "") + if cc: + info["phys"] = cc.split(":")[1].strip() + info["isa"] = {k: int(k in flags) for k in + ["avx2", "avx512f", "avx_vnni", "avx512_vnni", "amx_int8"]} + except Exception: + pass + return info + + +def platform_cpu(): + import platform + return platform.processor() or platform.machine() + + +def make_session(path, provider, intra, inter, profile=False): + so = ort.SessionOptions() + so.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL + if intra: + so.intra_op_num_threads = intra + if inter: + so.inter_op_num_threads = inter + so.enable_profiling = profile + if provider == "openvino": + providers = [("OpenVINOExecutionProvider", {"device_type": "CPU"}), "CPUExecutionProvider"] + else: + providers = ["CPUExecutionProvider"] + return ort.InferenceSession(str(path), sess_options=so, providers=providers) + + +def dim_value(name, axis, ndim, meta, seq): + n = name.lower() + if axis == 0 and ndim >= 2: + return 1 + if "audio" in n: + return int(meta.get("ssl_in_16k", seq)) + if "local" in n or ("ssl_features" in n and "global" not in n): + return int(meta.get("enc_tokens", 1) * meta.get("downsample_factor", 1)) or seq + if "token" in n or "indices" in n: + return int(meta.get("dec_tokens", seq)) + return seq + + +def resolve_inputs(sess, meta, seq, rng): + feeds, shapes = {}, {} + for inp in sess.get_inputs(): + dt = NP.get(inp.type, np.float32) + shape = [d if isinstance(d, int) and d > 0 + else dim_value(inp.name, ax, len(inp.shape), meta, seq) + for ax, d in enumerate(inp.shape)] + shapes[inp.name] = (shape, TAG.get(inp.type, "?")) + n = inp.name.lower() + if np.issubdtype(dt, np.integer): + feeds[inp.name] = np.zeros(shape, dtype=dt) + elif dt == np.bool_: + feeds[inp.name] = np.ones(shape, dtype=dt) + else: + a = rng.standard_normal(shape).astype(dt) + if "std" in n: + a = np.abs(a) + 1.0 + elif "mean" in n: + a *= 0.0 + elif "audio" in n: + a *= 0.1 + feeds[inp.name] = a + return feeds, shapes + + +def bench(sess, feeds, runs, warmup): + out = [o.name for o in sess.get_outputs()] + for _ in range(warmup): + sess.run(out, feeds) + ts = [] + for _ in range(runs): + t = time.perf_counter() + sess.run(out, feeds) + ts.append((time.perf_counter() - t) * 1e3) + return ts, out + + +def profile_ops(path, provider, intra, inter, feeds, out, runs): + sess = make_session(path, provider, intra, inter, profile=True) + for _ in range(runs): + sess.run(out, feeds) + prof = Path(sess.end_profiling()) + events = json.loads(prof.read_text()) + prof.unlink(missing_ok=True) + agg, prov = {}, {} + for e in events: + if e.get("cat") == "Node" and e.get("name", "").endswith("kernel_time"): + op = e.get("args", {}).get("op_name", "?") + agg[op] = agg.get(op, 0.0) + e.get("dur", 0) + p = e.get("args", {}).get("provider", "") + if p: + prov.setdefault(op, set()).add(p) + rows = sorted(agg.items(), key=lambda kv: kv[1], reverse=True) + return rows, (sum(agg.values()) or 1.0), prov + + +def static_ops(path): + try: + import onnx + except Exception: + return None + m = onnx.load(str(path), load_external_data=False) + c = {} + for node in m.graph.node: + c[node.op_type] = c.get(node.op_type, 0) + 1 + return dict(sorted(c.items(), key=lambda kv: kv[1], reverse=True)) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--dir", default="outputs") + ap.add_argument("--meta") + ap.add_argument("--provider", choices=["cpu", "openvino"], default="cpu") + ap.add_argument("--intra", type=int, default=0) + ap.add_argument("--inter", type=int, default=0) + ap.add_argument("--runs", type=int, default=50) + ap.add_argument("--warmup", type=int, default=5) + ap.add_argument("--seq", type=int, default=100) + ap.add_argument("--extra", nargs="*", default=[]) + ap.add_argument("--quant", action="store_true") + args = ap.parse_args() + + d = Path(args.dir) + meta = json.loads(Path(args.meta or d / "meta.json").read_text()) + rng = np.random.default_rng(0) + avail = ort.get_available_providers() + prov = args.provider + if prov == "openvino" and "OpenVINOExecutionProvider" not in avail: + prov = "cpu" + ov_note = "requested but NOT installed -> fell back to cpu" + else: + ov_note = "available" if "OpenVINOExecutionProvider" in avail else "not installed" + + models = {g: d / f"{g}.onnx" for g in GRAPHS if (d / f"{g}.onnx").exists()} + if args.quant: + for g in GRAPHS: + q = d / f"{g}_quant.onnx" + if q.exists(): + models[f"{g}_q"] = q + for kv in args.extra: + name, _, path = kv.partition("=") + models[name] = Path(path) + + ci = cpu_info() + isa = " ".join(f"{k}={v}" for k, v in ci["isa"].items()) + print("=== ENV ===") + print(f"cpu: {ci['cpu']}") + print(f"cores: {ci['phys']} phys / {ci['logical']} logical") + print(f"isa: {isa}") + print(f"onnxruntime: {ort.__version__}") + print(f"providers avail: {avail}") + print(f"openvino EP: {ov_note}") + print(f"config: provider={prov} intra={args.intra or 'default'} " + f"inter={args.inter or 'default'} runs={args.runs}") + + med = {} + csv_rows = [("graph", "med_ms", "mean_ms", "p90_ms", "min_ms", "runs")] + op_rows = [("graph", "op", "ms_per_run", "pct", "provider")] + for name, path in models.items(): + print(f"\n=== {name.upper()} ===") + print(f"path: {path} size: {path.stat().st_size / 1e6:.3g} MB") + try: + sess = make_session(path, prov, args.intra, args.inter) + feeds, shapes = resolve_inputs(sess, meta, args.seq, rng) + print("inputs: " + " ".join( + f"{k}[{','.join(map(str, s))}]{t}" for k, (s, t) in shapes.items())) + ts, out = bench(sess, feeds, args.runs, args.warmup) + m = statistics.median(ts) + med[name] = m + p90 = sorted(ts)[int(0.9 * len(ts)) - 1] + print(f"latency ms: med {m:.3g} mean {statistics.fmean(ts):.3g} " + f"p90 {p90:.3g} min {min(ts):.3g}") + csv_rows.append((name, f"{m:.3g}", f"{statistics.fmean(ts):.3g}", + f"{p90:.3g}", f"{min(ts):.3g}", args.runs)) + + so = static_ops(path) + if so: + print("ops static: " + " ".join(f"{k}:{v}" for k, v in list(so.items())[:10])) + + rows, total, pmap = profile_ops(path, prov, args.intra, args.inter, feeds, out, args.warmup or 5) + multi = len({p for ps in pmap.values() for p in ps}) > 1 + parts = [] + for op, dur in rows[:6]: + pr = "/".join(sorted(x.replace("ExecutionProvider", "") for x in pmap.get(op, []))) + tag = f"({pr})" if multi else "" + parts.append(f"{op}{tag} {dur / (args.warmup or 5) / 1e3:.3g}ms {100 * dur / total:.0f}%") + op_rows.append((name, op, f"{dur / (args.warmup or 5) / 1e3:.3g}", + f"{100 * dur / total:.0f}", pr or "CPU")) + print("ops time: " + " | ".join(parts)) + except Exception as e: + print(f"FAILED: {e}") + + print("\n=== ROLLUP ===") + ds = meta.get("downsample_factor", 1) + tok16 = ds * meta.get("wavlm_hop", 1) + sr16 = meta.get("ssl_sample_rate", 16000) + chunk = meta.get("chunk", 1) + audio_s = chunk * tok16 / sr16 + per_win = sum(med.get(g, 0.0) for g in ("ssl", "encode", "decode")) + print(f"chunk={chunk} tok16={tok16} audio/window={audio_s * 1e3:.3g}ms") + print(f"per-window compute (ssl+encode+decode): {per_win:.3g}ms") + if audio_s > 0: + print(f"est streaming RTF: {(per_win / 1e3) / audio_s:.3g} (global enc one-shot, excluded)") + + if args.quant: + print("fp32 -> quant:") + for g in ("ssl", "encode", "decode", "global"): + if g in med and f"{g}_q" in med: + f0, f1 = med[g], med[f"{g}_q"] + print(f" {g}: {f0:.3g} -> {f1:.3g}ms ({100 * (1 - f1 / f0):+.0f}%)") + per_q = sum(med.get(f"{g}_q", med.get(g, 0.0)) for g in ("ssl", "encode", "decode")) + if audio_s > 0: + print(f"per-window quant: {per_q:.3g}ms RTF {(per_q / 1e3) / audio_s:.3g}") + + od = Path("outputs") + od.mkdir(exist_ok=True) + import csv + with open(od / "bench.csv", "w", newline="") as f: + csv.writer(f).writerows(csv_rows) + with open(od / "ops.csv", "w", newline="") as f: + csv.writer(f).writerows(op_rows) + print(f"\nwrote {od/'bench.csv'} {od/'ops.csv'}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/live_onnx.py b/live_onnx.py index 66ddb38..743ec21 100644 --- a/live_onnx.py +++ b/live_onnx.py @@ -8,7 +8,7 @@ import json import numpy as np import onnxruntime as ort -# This bullshit + ort.preload_dlls() import sounddevice as sd import soundfile as sf @@ -100,11 +100,16 @@ class StreamingVCONNX: opts = ort.SessionOptions() opts.inter_op_num_threads = 1 - opts.intra_op_num_threads = 1 + opts.intra_op_num_threads = 0 opts.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL opts.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL - prov = ["CUDAExecutionProvider", "CPUExecutionProvider"] if args.cuda else ["CPUExecutionProvider"] + if getattr(args, "openvino", False): + prov = [("OpenVINOExecutionProvider", {"device_type": "CPU"}), "CPUExecutionProvider"] + elif args.cuda: + prov = ["CUDAExecutionProvider", "CPUExecutionProvider"] + else: + prov = ["CPUExecutionProvider"] self.ssl = ort.InferenceSession(args.ssl, sess_options=opts, providers=prov) self.enc = ort.InferenceSession(args.encode, sess_options=opts, providers=prov) @@ -118,7 +123,7 @@ class StreamingVCONNX: self.tokens = None self.decoded = 0 self.prev_local_feats = None - self.ema_alpha = 0.8 # Adjust between 0.5 (heavy smoothing) and 1.0 (no smoothing) + self.ema_alpha = 0.4 def _ssl(self, win16): w = take(win16, 0, self.ssl_in).reshape(1, -1) @@ -167,15 +172,11 @@ class StreamingVCONNX: self.tokens = None self.decoded = 0 - def _encode_window(self, win16): - local_feats, _ = self._ssl(win16) - - # Apply temporal smoothing to the continuous representations + def apply_ema(self, local_feats): if self.prev_local_feats is not None and local_feats.shape == self.prev_local_feats.shape: local_feats = self.ema_alpha * local_feats + (1.0 - self.ema_alpha) * self.prev_local_feats - self.prev_local_feats = local_feats.copy() - return self._encode(local_feats, self.src_mean, self.src_std) + return local_feats def _decode(self, win_tokens, keep_left, keep_n): real, imag = self.dec.run( @@ -232,18 +233,17 @@ def main(): parser.add_argument("--list-devices", action="store_true") parser.add_argument("--input", type=int) parser.add_argument("--output", type=int) - parser.add_argument("--target", type=Path, required=True, help="Target voice reference WAV") - parser.add_argument("--seed-audio", type=Path, help="Seed speaker calibration WAV (optional)") - parser.add_argument("--encode", required=True, help="Path to encode.onnx") - parser.add_argument("--decode", help="Path to decode.onnx (defaults to encode.onnx parent folder)") - parser.add_argument("--global", dest="global_path", help="Path to global.onnx (defaults to encode.onnx parent folder)") - parser.add_argument("--ssl", help="Path to ssl.onnx (defaults to encode.onnx parent folder)") - parser.add_argument("--meta", help="Path to meta.json (defaults to encode.onnx parent folder)") - parser.add_argument("--cuda", action="store_true", help="Enable CUDA execution provider") - parser.add_argument("--rms-floor", type=float, default=0.0035, - help="RMS threshold below which audio chunk is evaluated as silence") - parser.add_argument("--hangover-chunks", type=int, default=3, - help="Number of chunks to hold the gate open after RMS drop to prevent trailing cutoffs") + parser.add_argument("--target", type=Path, required=True) + parser.add_argument("--seed-audio", type=Path) + parser.add_argument("--encode", required=True) + parser.add_argument("--decode") + parser.add_argument("--global", dest="global_path") + parser.add_argument("--ssl") + parser.add_argument("--meta") + parser.add_argument("--cuda", action="store_true") + parser.add_argument("--openvino", action="store_true") + parser.add_argument("--rms-floor", type=float, default=0.0035) + parser.add_argument("--hangover-chunks", type=int, default=3) args = parser.parse_args() if args.list_devices: @@ -265,24 +265,19 @@ def main(): sr = vc.sr sr16 = vc.sr16 - # Calculate sample sizes based on target (playback) sample rate - # token_hz is standard (usually 25 Hz), tok_samples is usually 1764 for 44.1 kHz token_hz = meta["token_hz"] tok_samples = sr // token_hz chunk_samples = vc.chunk * tok_samples budget_ms = (vc.chunk / token_hz) * 1000 - # Calculated parameters for processing 16 kHz streams tok16 = vc.tok16 chunk_samples_16k = vc.chunk * tok16 left_pad_16k = vc.enc_left * tok16 right_pad_16k = vc.enc_right * tok16 - ssl_in_16k = vc.ssl_in print(f"Sample Rate: {sr} Hz (target) | 16000 Hz (SSL internal)") print(f"Chunk Size: {vc.chunk} tokens ({budget_ms:.1f}ms budget)") - print(f"Loading target speaker profile: {args.target}...") target_audio = load_16k(args.target, sr16) vc.set_target(target_audio) @@ -290,29 +285,22 @@ def main(): n_in_ch = min(in_info["max_input_channels"], 2) if args.seed_audio: - print(f"Loading speaker calibration profile: {args.seed_audio}...") seed_audio = load_16k(args.seed_audio, sr16) else: - print("\n" + "=" * 60) - print("No seed-audio provided. Recording 3 seconds for normalization calibration.") - print("Please speak into your microphone...") - print("=" * 60) recorded = sd.rec(int(3.0 * sr), samplerate=sr, channels=n_in_ch, dtype="float32") sd.wait() - print("Recording complete. Calibrating feature scaling...") recorded_mono = recorded.mean(axis=1) if recorded.shape[1] > 1 else recorded[:, 0] seed_audio = resample(recorded_mono, sr, sr16) - print("Seeding streaming context from speaker profile...") vc.seed(seed_audio) - # Establish initial left-side padding context buffer in 16 kHz if len(seed_audio) >= left_pad_16k: raw_input_accum_16k = seed_audio[-left_pad_16k:] else: raw_input_accum_16k = np.pad(seed_audio, (left_pad_16k - len(seed_audio), 0)) in_q = queue.Queue(maxsize=8) + ssl_q = queue.Queue(maxsize=8) out_q = queue.Queue(maxsize=2) stop_event = threading.Event() @@ -330,12 +318,58 @@ def main(): except queue.Empty: continue - print(f"\n{'chunk':>6} {'q_in':>4} {'q_out':>5} {'enc':>7} {'dec':>7} {'total':>7} {'budget':>7} {'gap':>7}") - print("-" * 76) + def ssl_thread_func(accum_16k): + hangover_counter = 0 + t_last = None + while not stop_event.is_set(): + try: + raw = in_q.get(timeout=0.5) + except queue.Empty: + continue + + t_now = time.perf_counter() + gap_ms = (t_now - t_last) * 1000 if t_last else 0.0 + t_last = t_now + + rms = float(np.sqrt(np.mean(raw ** 2))) + + if rms >= args.rms_floor: + hangover_counter = args.hangover_chunks + is_silence = False + else: + if hangover_counter > 0: + hangover_counter -= 1 + is_silence = False + else: + is_silence = True + + raw_16k = resample(raw, sr, sr16) + accum_16k = np.concatenate([accum_16k, raw_16k]) + required_samples_16k = left_pad_16k + chunk_samples_16k + right_pad_16k + + if len(accum_16k) >= required_samples_16k: + window_16k = accum_16k[:required_samples_16k] + accum_16k = accum_16k[chunk_samples_16k:] + + fade_len = int(0.01 * sr16) + ramp_down = np.linspace(1.0, 0.0, fade_len, dtype=np.float32) + + if is_silence: + window_16k = window_16k.copy() + active_start = left_pad_16k + active_end = left_pad_16k + chunk_samples_16k + window_16k[active_start : active_start + fade_len] *= ramp_down + window_16k[active_start + fade_len : active_end] = 0.0 + + local_feats, t_ssl = sync_time(lambda: vc._ssl(window_16k)[0]) + ssl_q.put((local_feats, is_silence, t_ssl, gap_ms, rms)) + else: + ssl_q.put((None, is_silence, 0.0, gap_ms, rms)) + + print(f"\n{'chunk':>6} {'q_in':>4} {'q_ss':>4} {'q_out':>5} {'ssl':>7} {'enc':>7} {'dec':>7} {'total':>7} {'budget':>7} {'gap':>7}") + print("-" * 88) chunk_n = 0 - t_last = None - hangover_counter = 0 with sd.InputStream(device=args.input, channels=n_in_ch, samplerate=sr, blocksize=chunk_samples, dtype="float32", @@ -344,54 +378,23 @@ def main(): dtype="float32", latency="low") as out_stream: writer = threading.Thread(target=write_thread, args=(out_stream,), daemon=True) + ssl_worker = threading.Thread(target=ssl_thread_func, args=(raw_input_accum_16k,), daemon=True) + writer.start() + ssl_worker.start() try: while True: - raw = in_q.get() - t_now = time.perf_counter() - gap_ms = (t_now - t_last) * 1000 if t_last else 0.0 - t_last = t_now + try: + item = ssl_q.get(timeout=0.5) + except queue.Empty: + continue - rms = float(np.sqrt(np.mean(raw ** 2))) + local_feats, is_silence, t_ssl, gap_ms, rms = item - if rms >= args.rms_floor: - hangover_counter = args.hangover_chunks - is_silence = False - else: - if hangover_counter > 0: - hangover_counter -= 1 - is_silence = False - else: - is_silence = True - - # Resample current input chunk to 16 kHz - raw_16k = resample(raw, sr, sr16) - raw_input_accum_16k = np.concatenate([raw_input_accum_16k, raw_16k]) - required_samples_16k = left_pad_16k + chunk_samples_16k + right_pad_16k - - if len(raw_input_accum_16k) >= required_samples_16k: - window_16k = raw_input_accum_16k[:required_samples_16k] - raw_input_accum_16k = raw_input_accum_16k[chunk_samples_16k:] - - # Create a simple linear ramp at the beginning of your script or class - fade_len = int(0.01 * sr16) # 10ms ramp - ramp_down = np.linspace(1.0, 0.0, fade_len, dtype=np.float32) - ramp_up = np.linspace(0.0, 1.0, fade_len, dtype=np.float32) - - # Apply a soft gate instead of hard zeroing - if is_silence: - window_16k = window_16k.copy() - # Smoothly ramp down the boundary before zeroing - active_start = left_pad_16k - active_end = left_pad_16k + chunk_samples_16k - - # Apply fade out - window_16k[active_start : active_start + fade_len] *= ramp_down - window_16k[active_start + fade_len : active_end] = 0.0 - - # Run inference via ONNX models - idx, t_enc = sync_time(lambda: vc._encode_window(window_16k)) + if local_feats is not None: + local_feats = vc.apply_ema(local_feats) + idx, t_enc = sync_time(lambda: vc._encode(local_feats, vc.src_mean, vc.src_std)) chunk_tokens = idx[vc.enc_left : vc.enc_left + vc.chunk] vc._commit_tokens(chunk_tokens) audio_out, t_dec = sync_time(lambda: vc._drain(final=False)) @@ -407,19 +410,19 @@ def main(): out_q.put(pcm_out) - total = t_enc + t_dec + total = t_ssl + t_enc + t_dec chunk_n += 1 if is_silence: print( - f"{chunk_n:>6} {in_q.qsize():>4} {out_q.qsize():>5} " - f"{'--silence--':>31} rms={rms:.4f}", + f"{chunk_n:>6} {in_q.qsize():>4} {ssl_q.qsize():>4} {out_q.qsize():>5} " + f"{'--silence--':>54} rms={rms:.4f}", flush=True, ) else: print( - f"{chunk_n:>6} {in_q.qsize():>4} {out_q.qsize():>5} " - f"{t_enc:>6.1f}ms {t_dec:>6.1f}ms " + f"{chunk_n:>6} {in_q.qsize():>4} {ssl_q.qsize():>4} {out_q.qsize():>5} " + f"{t_ssl:>6.1f}ms {t_enc:>6.1f}ms {t_dec:>6.1f}ms " f"{total:>6.1f}ms {budget_ms:>6.0f}ms {gap_ms:>6.1f}ms", flush=True, ) @@ -429,6 +432,7 @@ def main(): finally: stop_event.set() writer.join() + ssl_worker.join() print("stopped") diff --git a/pyproject.toml b/pyproject.toml index bc0832b..6e22fa9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,8 +9,11 @@ dependencies = [ "numpy>=2.4.6", "onnxruntime>=1.26.0", "onnxruntime-gpu>=1.26.0", + "onnxruntime-openvino>=1.24.1", + "onnxruntime-tools>=1.7.0", "onnxscript>=0.7.0", "sounddevice>=0.5.5", + "sympy>=1.14.0", "torch>=2.11.0", ] diff --git a/quantize.py b/quantize.py index 0273d64..7ef61a4 100644 --- a/quantize.py +++ b/quantize.py @@ -1,44 +1,168 @@ import argparse +import json from pathlib import Path + +import numpy as np +import onnx +import onnxruntime as ort from onnxruntime.quantization import quantize_dynamic, QuantType from onnxruntime.quantization.shape_inference import quant_pre_process +ort.set_default_logger_severity(3) +OPS = ["Conv", "Gemm", "MatMul"] -def quantize_model(input_path: Path, output_path: Path): - # Create temporary path for the pre-processed model - preprocessed_path = input_path.with_name(f"{input_path.stem}_preprocessed.onnx") - print(f"Pre-processing {input_path.name}...") +def has_external(path): + m = onnx.load(str(path), load_external_data=False) + return any(t.data_location == onnx.TensorProto.EXTERNAL for t in m.graph.initializer) + + +def grouped_or_nonconst_convs(path): + m = onnx.load(str(path), load_external_data=False) + inits = {i.name for i in m.graph.initializer} + bad = [] + for n in m.graph.node: + if n.op_type != "Conv": + continue + group = next((a.i for a in n.attribute if a.name == "group"), 1) + w_const = len(n.input) > 1 and n.input[1] in inits + if group > 1 or not w_const: + bad.append(n.name) + return bad + + +def quantize_one(path, weight_type, reduce_range): + stem = path.stem + out = path.with_name(f"{stem}_quant.onnx") + pre = path.with_name(f"{stem}_pre.onnx") + target = path try: - quant_pre_process(str(input_path), str(preprocessed_path)) - target_input = preprocessed_path - except Exception as e: - print(f"Pre-processing skipped or failed: {e}") - target_input = input_path + quant_pre_process(str(path), str(pre), skip_optimization=False, + skip_onnx_shape=False, skip_symbolic_shape=False, auto_merge=True) + target = pre + except Exception as e1: + try: + quant_pre_process(str(path), str(pre), skip_optimization=False, + skip_onnx_shape=False, skip_symbolic_shape=True) + target = pre + print(" preprocess: symbolic shape skipped") + except Exception as e2: + print(f" preprocess failed, quantizing raw: {e2}") - print(f"Quantizing {target_input.name}...") - try: - quantize_dynamic( - model_input=str(target_input), - model_output=str(output_path), - weight_type=QuantType.QUInt8, - # Limit quantization to MatMul. This bypasses the Conv layers - # that cause weight initialization errors, while still optimizing - # the heavy transformer layers. - op_types_to_quantize=["MatMul"] - ) - print(f"Quantization complete: {output_path}") - finally: - # Clean up temporary preprocessed file if it was created - if preprocessed_path.exists() and preprocessed_path != input_path: - preprocessed_path.unlink() + exclude = grouped_or_nonconst_convs(target) if stem == "ssl" else [] + if exclude: + print(f" excluding {len(exclude)} grouped/non-const conv(s)") + + quantize_dynamic( + model_input=str(target), model_output=str(out), + weight_type=weight_type, op_types_to_quantize=OPS, + nodes_to_exclude=exclude, reduce_range=reduce_range, + ) + pre.unlink(missing_ok=True) + b = out.stat().st_size + if has_external(path): + print(f" {path.name} -> {out.name} {b/1e6:.3g} MB int8 self-contained (fp32 weights were external)") + else: + a = path.stat().st_size + print(f" {path.name} -> {out.name} {a/1e6:.3g} -> {b/1e6:.3g} MB ({100*(1-b/a):.0f}% smaller)") + return out + + +def feeds_for(sess, meta, rng): + feeds = {} + for inp in sess.get_inputs(): + dt = np.int64 if "int64" in inp.type else (np.int32 if "int32" in inp.type else np.float32) + shape = [d if isinstance(d, int) and d > 0 + else (1 if ax == 0 and len(inp.shape) >= 2 else meta.get("enc_ssl_frames", 100)) + for ax, d in enumerate(inp.shape)] + n = inp.name.lower() + if np.issubdtype(dt, np.integer): + feeds[inp.name] = np.zeros(shape, dtype=dt) + else: + a = rng.standard_normal(shape).astype(np.float32) + feeds[inp.name] = (np.abs(a) + 0.5) if "std" in n else (a * 0.0 if "mean" in n else a) + return feeds + + +def check(fp32, quant, meta): + rng = np.random.default_rng(0) + s0 = ort.InferenceSession(str(fp32), providers=["CPUExecutionProvider"]) + s1 = ort.InferenceSession(str(quant), providers=["CPUExecutionProvider"]) + feeds = feeds_for(s0, meta, rng) + out = [o.name for o in s0.get_outputs()] + r0 = s0.run(out, feeds) + r1 = s1.run(out, feeds) + for name, a, b in zip(out, r0, r1): + if np.issubdtype(a.dtype, np.integer): + print(f" {name}: {100*(a != b).mean():.2f}% tokens changed") + else: + d = np.abs(a - b) + print(f" {name}: max|d|={d.max():.3g} mean|d|={d.mean():.3g}") + + +def check_real(d, meta, source, target): + import infer + a = argparse.Namespace(ssl=str(d / "ssl.onnx"), encode=str(d / "encode.onnx"), + decode=str(d / "decode.onnx"), global_path=str(d / "global.onnx"), + cuda=False) + vc = infer.Infer(a, meta) + sr16 = meta["ssl_sample_rate"] + src16 = infer.load_16k(source, sr16) + mean, std, _ = vc.calibrate(src16) + qs = {n: ort.InferenceSession(str(d / f"{n}_quant.onnx"), providers=["CPUExecutionProvider"]) + for n in ["ssl", "encode", "decode", "global"] if (d / f"{n}_quant.onnx").exists()} + + keep, win = next(vc._windows(src16)) + win1 = infer.take(win, 0, vc.ssl_in).reshape(1, -1) + local_real = vc._ssl(win)[0] + if "ssl" in qs: + l1, g1 = qs["ssl"].run(["local_features", "global_features"], {"audio_16k": win1}) + l0, g0 = vc.ssl.run(["local_features", "global_features"], {"audio_16k": win1}) + print(f" ssl local max|d|={np.abs(l0 - l1).max():.3g} global max|d|={np.abs(g0 - g1).max():.3g}") + + if "encode" in qs: + feed = {"local_ssl_features": local_real, "mean": mean, "std": std} + t0 = vc.enc.run(["content_token_indices"], feed)[0] + t1 = qs["encode"].run(["content_token_indices"], feed)[0] + k = slice(vc.enc_left, vc.enc_left + keep) + print(f" encode tokens (real, center {keep}): {100 * (t0[k] == t1[k]).mean():.1f}% agree") + + if "decode" in qs and target: + emb = vc.embed(infer.load_16k(target, sr16)) + toks = vc.tokens(src16, mean, std) + lo = vc.dec_left + w = toks[np.clip(np.arange(lo, lo + vc.dec_tokens), 0, len(toks) - 1)].astype(np.int64) + feed = {"content_token_indices": w, "global_embedding": emb} + r0 = vc.dec.run(["spec_real", "spec_imag"], feed) + r1 = qs["decode"].run(["spec_real", "spec_imag"], feed) + print(f" decode spec_real max|d|={np.abs(r0[0] - r1[0]).max():.3g} " + f"spec_imag max|d|={np.abs(r0[1] - r1[1]).max():.3g}") if __name__ == "__main__": p = argparse.ArgumentParser() - p.add_argument("--model", required=True, help="Path to the ONNX model to quantize") + p.add_argument("--dir", default="outputs") + p.add_argument("--models", nargs="*", default=["ssl", "encode", "decode", "global"]) + p.add_argument("--weight-type", choices=["int8", "uint8"], default="int8") + p.add_argument("--no-reduce-range", action="store_true") + p.add_argument("--check", action="store_true") + p.add_argument("--source") + p.add_argument("--target") args = p.parse_args() - in_path = Path(args.model) - out_path = in_path.with_name(f"{in_path.stem}_quant.onnx") - quantize_model(in_path, out_path) \ No newline at end of file + d = Path(args.dir) + wt = QuantType.QInt8 if args.weight_type == "int8" else QuantType.QUInt8 + meta = json.loads((d / "meta.json").read_text()) if (d / "meta.json").exists() else {} + + for name in args.models: + f = d / f"{name}.onnx" + if not f.exists(): + continue + print(f"{name}:") + q = quantize_one(f, wt, not args.no_reduce_range) + if args.check: + check(f, q, meta) + + if args.source: + print("real-audio check:") + check_real(d, meta, args.source, args.target) \ No newline at end of file diff --git a/uv.lock b/uv.lock index 934b8cf..ee8b74a 100644 --- a/uv.lock +++ b/uv.lock @@ -121,6 +121,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, ] +[[package]] +name = "coloredlogs" +version = "15.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "humanfriendly" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cc/c7/eed8f27100517e8c0e6b923d5f0845d0cb99763da6fdee00478f91db7325/coloredlogs-15.0.1.tar.gz", hash = "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0", size = 278520, upload-time = "2021-06-11T10:22:45.202Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/06/3d6badcf13db419e25b07041d9c7b4a2c331d3f4e7134445ec5df57714cd/coloredlogs-15.0.1-py2.py3-none-any.whl", hash = "sha256:612ee75c546f53e92e70049c9dbfcc18c935a2b9a53b66085ce9ef6a6e5c0934", size = 46018, upload-time = "2021-06-11T10:22:42.561Z" }, +] + [[package]] name = "cuda-bindings" version = "12.9.7" @@ -210,8 +222,11 @@ dependencies = [ { name = "numpy" }, { name = "onnxruntime" }, { name = "onnxruntime-gpu" }, + { name = "onnxruntime-openvino" }, + { name = "onnxruntime-tools" }, { name = "onnxscript" }, { name = "sounddevice" }, + { name = "sympy" }, { name = "torch", version = "2.11.0+cu128", source = { registry = "https://download.pytorch.org/whl/cu128" }, marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "torch", version = "2.12.0", source = { registry = "https://pypi.org/simple" }, marker = "sys_platform != 'linux' and sys_platform != 'win32'" }, ] @@ -222,8 +237,11 @@ requires-dist = [ { name = "numpy", specifier = ">=2.4.6" }, { name = "onnxruntime", specifier = ">=1.26.0" }, { name = "onnxruntime-gpu", specifier = ">=1.26.0" }, + { name = "onnxruntime-openvino", specifier = ">=1.24.1" }, + { name = "onnxruntime-tools", specifier = ">=1.7.0" }, { name = "onnxscript", specifier = ">=0.7.0" }, { name = "sounddevice", specifier = ">=0.5.5" }, + { name = "sympy", specifier = ">=1.14.0" }, { name = "torch", specifier = ">=2.11.0" }, ] @@ -352,6 +370,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/02/28/d7cef5e477b855c25d415b8f57e5bc7347c7a90cad3acf1725d0c92ca294/huggingface_hub-1.17.0-py3-none-any.whl", hash = "sha256:3b8156d23118e87f6a587648bfbc04f04a12a757ccb4ed298b35c4ae638bf24c", size = 671546, upload-time = "2026-05-28T15:12:11.441Z" }, ] +[[package]] +name = "humanfriendly" +version = "10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyreadline3", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cc/3f/2c29224acb2e2df4d2046e4c73ee2662023c58ff5b113c4c1adac0886c43/humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc", size = 360702, upload-time = "2021-09-17T21:40:43.31Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f0/0f/310fb31e39e2d734ccaa2c0fb981ee41f7bd5056ce9bc29b2248bd569169/humanfriendly-10.0-py2.py3-none-any.whl", hash = "sha256:1697e1a8a8f550fd43c2865cd84542fc175a61dcb779b6fee18cf6b6ccba1477", size = 86794, upload-time = "2021-09-17T21:40:39.897Z" }, +] + [[package]] name = "idna" version = "3.17" @@ -886,6 +916,42 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/97/91/93ffe5431d154989f5e04864a25a97eea480997d771232bcbbc538188241/onnxruntime_gpu-1.26.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:56dc7b73954ff4bdc71f5b8ab306b6f61be5d007881b6ef423a609e2b9cd088b", size = 276991545, upload-time = "2026-05-08T19:16:33.347Z" }, ] +[[package]] +name = "onnxruntime-openvino" +version = "1.24.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "flatbuffers" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "protobuf" }, + { name = "sympy" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/50/cf/17ba72de2df0fcba349937d2788f154397bbc2d1a2d67772a97e26f6bc5f/onnxruntime_openvino-1.24.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:d617fac2f59a6ab5ea59a788c3e1592240a129642519aaeaa774761dfe35150e", size = 84433207, upload-time = "2026-02-26T13:44:41.395Z" }, + { url = "https://files.pythonhosted.org/packages/59/37/d301f2c68b19a9485ed5db3047e0fb52478f3e73eb08c7d2a7c61be7cc1c/onnxruntime_openvino-1.24.1-cp312-cp312-win_amd64.whl", hash = "sha256:f186335a9c9b255633275290da7521d3d4d14c7773fee3127bfa040234d3fa5a", size = 13658075, upload-time = "2026-02-26T13:44:44.905Z" }, + { url = "https://files.pythonhosted.org/packages/08/07/f225999919f56506b603aaa3ff837ad563ab26f86906ed7fa7e5abcd849e/onnxruntime_openvino-1.24.1-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:2c3bb73e68ac27f4891af8a595c1faf574ec68b772e6583c90a0b997a1822782", size = 84433183, upload-time = "2026-02-26T13:44:50.254Z" }, + { url = "https://files.pythonhosted.org/packages/3e/92/46ae2cd565961a89189900f385bb2f13a9fa731ea4674001d23720fbb1e0/onnxruntime_openvino-1.24.1-cp313-cp313-win_amd64.whl", hash = "sha256:434bf49aa71393c577a456c9d76c98e6d6958a833fa0876793e3d5437b5a511a", size = 13658485, upload-time = "2026-02-26T13:44:53.889Z" }, +] + +[[package]] +name = "onnxruntime-tools" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coloredlogs" }, + { name = "numpy" }, + { name = "onnx" }, + { name = "packaging" }, + { name = "psutil" }, + { name = "py-cpuinfo" }, + { name = "py3nvml" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/fd/b5/c36283fef3b1d492a39d1b5f3f195965fbf002b168633daad302c51d8f4c/onnxruntime_tools-1.7.0.tar.gz", hash = "sha256:6dbdcee49424e066bcd10357c37d51bc422ae26494e3c2f0c1970d534f967f6d", size = 141435, upload-time = "2021-03-25T21:42:42.571Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6f/b0/db0e73356df0aaa8737e6f13c0dac499b5d904d3fa267c8ebf24515e8001/onnxruntime_tools-1.7.0-py3-none-any.whl", hash = "sha256:1dff888b5c482ac5bc627f12e108445fefcb3d600c43f63633975316fe617ad8", size = 212695, upload-time = "2021-03-25T21:42:40.551Z" }, +] + [[package]] name = "onnxscript" version = "0.7.0" @@ -927,6 +993,55 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b8/ef/50433d346c56657a70d27f156c7b349ac59a068b01de4eb796e747eecc43/protobuf-7.35.0-py3-none-any.whl", hash = "sha256:c13f325cf242bad135c350629eeb5d54b24228eb472fb3e2e9ebbd4c5dc20ca0", size = 171659, upload-time = "2026-05-19T23:02:27.842Z" }, ] +[[package]] +name = "psutil" +version = "7.2.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/c6/d1ddf4abb55e93cebc4f2ed8b5d6dbad109ecb8d63748dd2b20ab5e57ebe/psutil-7.2.2.tar.gz", hash = "sha256:0746f5f8d406af344fd547f1c8daa5f5c33dbc293bb8d6a16d80b4bb88f59372", size = 493740, upload-time = "2026-01-28T18:14:54.428Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/08/510cbdb69c25a96f4ae523f733cdc963ae654904e8db864c07585ef99875/psutil-7.2.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:2edccc433cbfa046b980b0df0171cd25bcaeb3a68fe9022db0979e7aa74a826b", size = 130595, upload-time = "2026-01-28T18:14:57.293Z" }, + { url = "https://files.pythonhosted.org/packages/d6/f5/97baea3fe7a5a9af7436301f85490905379b1c6f2dd51fe3ecf24b4c5fbf/psutil-7.2.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:e78c8603dcd9a04c7364f1a3e670cea95d51ee865e4efb3556a3a63adef958ea", size = 131082, upload-time = "2026-01-28T18:14:59.732Z" }, + { url = "https://files.pythonhosted.org/packages/37/d6/246513fbf9fa174af531f28412297dd05241d97a75911ac8febefa1a53c6/psutil-7.2.2-cp313-cp313t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1a571f2330c966c62aeda00dd24620425d4b0cc86881c89861fbc04549e5dc63", size = 181476, upload-time = "2026-01-28T18:15:01.884Z" }, + { url = "https://files.pythonhosted.org/packages/b8/b5/9182c9af3836cca61696dabe4fd1304e17bc56cb62f17439e1154f225dd3/psutil-7.2.2-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:917e891983ca3c1887b4ef36447b1e0873e70c933afc831c6b6da078ba474312", size = 184062, upload-time = "2026-01-28T18:15:04.436Z" }, + { url = "https://files.pythonhosted.org/packages/16/ba/0756dca669f5a9300d0cbcbfae9a4c30e446dfc7440ffe43ded5724bfd93/psutil-7.2.2-cp313-cp313t-win_amd64.whl", hash = "sha256:ab486563df44c17f5173621c7b198955bd6b613fb87c71c161f827d3fb149a9b", size = 139893, upload-time = "2026-01-28T18:15:06.378Z" }, + { url = "https://files.pythonhosted.org/packages/1c/61/8fa0e26f33623b49949346de05ec1ddaad02ed8ba64af45f40a147dbfa97/psutil-7.2.2-cp313-cp313t-win_arm64.whl", hash = "sha256:ae0aefdd8796a7737eccea863f80f81e468a1e4cf14d926bd9b6f5f2d5f90ca9", size = 135589, upload-time = "2026-01-28T18:15:08.03Z" }, + { url = "https://files.pythonhosted.org/packages/81/69/ef179ab5ca24f32acc1dac0c247fd6a13b501fd5534dbae0e05a1c48b66d/psutil-7.2.2-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:eed63d3b4d62449571547b60578c5b2c4bcccc5387148db46e0c2313dad0ee00", size = 130664, upload-time = "2026-01-28T18:15:09.469Z" }, + { url = "https://files.pythonhosted.org/packages/7b/64/665248b557a236d3fa9efc378d60d95ef56dd0a490c2cd37dafc7660d4a9/psutil-7.2.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7b6d09433a10592ce39b13d7be5a54fbac1d1228ed29abc880fb23df7cb694c9", size = 131087, upload-time = "2026-01-28T18:15:11.724Z" }, + { url = "https://files.pythonhosted.org/packages/d5/2e/e6782744700d6759ebce3043dcfa661fb61e2fb752b91cdeae9af12c2178/psutil-7.2.2-cp314-cp314t-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fa4ecf83bcdf6e6c8f4449aff98eefb5d0604bf88cb883d7da3d8d2d909546a", size = 182383, upload-time = "2026-01-28T18:15:13.445Z" }, + { url = "https://files.pythonhosted.org/packages/57/49/0a41cefd10cb7505cdc04dab3eacf24c0c2cb158a998b8c7b1d27ee2c1f5/psutil-7.2.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e452c464a02e7dc7822a05d25db4cde564444a67e58539a00f929c51eddda0cf", size = 185210, upload-time = "2026-01-28T18:15:16.002Z" }, + { url = "https://files.pythonhosted.org/packages/dd/2c/ff9bfb544f283ba5f83ba725a3c5fec6d6b10b8f27ac1dc641c473dc390d/psutil-7.2.2-cp314-cp314t-win_amd64.whl", hash = "sha256:c7663d4e37f13e884d13994247449e9f8f574bc4655d509c3b95e9ec9e2b9dc1", size = 141228, upload-time = "2026-01-28T18:15:18.385Z" }, + { url = "https://files.pythonhosted.org/packages/f2/fc/f8d9c31db14fcec13748d373e668bc3bed94d9077dbc17fb0eebc073233c/psutil-7.2.2-cp314-cp314t-win_arm64.whl", hash = "sha256:11fe5a4f613759764e79c65cf11ebdf26e33d6dd34336f8a337aa2996d71c841", size = 136284, upload-time = "2026-01-28T18:15:19.912Z" }, + { url = "https://files.pythonhosted.org/packages/e7/36/5ee6e05c9bd427237b11b3937ad82bb8ad2752d72c6969314590dd0c2f6e/psutil-7.2.2-cp36-abi3-macosx_10_9_x86_64.whl", hash = "sha256:ed0cace939114f62738d808fdcecd4c869222507e266e574799e9c0faa17d486", size = 129090, upload-time = "2026-01-28T18:15:22.168Z" }, + { url = "https://files.pythonhosted.org/packages/80/c4/f5af4c1ca8c1eeb2e92ccca14ce8effdeec651d5ab6053c589b074eda6e1/psutil-7.2.2-cp36-abi3-macosx_11_0_arm64.whl", hash = "sha256:1a7b04c10f32cc88ab39cbf606e117fd74721c831c98a27dc04578deb0c16979", size = 129859, upload-time = "2026-01-28T18:15:23.795Z" }, + { url = "https://files.pythonhosted.org/packages/b5/70/5d8df3b09e25bce090399cf48e452d25c935ab72dad19406c77f4e828045/psutil-7.2.2-cp36-abi3-manylinux2010_x86_64.manylinux_2_12_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:076a2d2f923fd4821644f5ba89f059523da90dc9014e85f8e45a5774ca5bc6f9", size = 155560, upload-time = "2026-01-28T18:15:25.976Z" }, + { url = "https://files.pythonhosted.org/packages/63/65/37648c0c158dc222aba51c089eb3bdfa238e621674dc42d48706e639204f/psutil-7.2.2-cp36-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b0726cecd84f9474419d67252add4ac0cd9811b04d61123054b9fb6f57df6e9e", size = 156997, upload-time = "2026-01-28T18:15:27.794Z" }, + { url = "https://files.pythonhosted.org/packages/8e/13/125093eadae863ce03c6ffdbae9929430d116a246ef69866dad94da3bfbc/psutil-7.2.2-cp36-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:fd04ef36b4a6d599bbdb225dd1d3f51e00105f6d48a28f006da7f9822f2606d8", size = 148972, upload-time = "2026-01-28T18:15:29.342Z" }, + { url = "https://files.pythonhosted.org/packages/04/78/0acd37ca84ce3ddffaa92ef0f571e073faa6d8ff1f0559ab1272188ea2be/psutil-7.2.2-cp36-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:b58fabe35e80b264a4e3bb23e6b96f9e45a3df7fb7eed419ac0e5947c61e47cc", size = 148266, upload-time = "2026-01-28T18:15:31.597Z" }, + { url = "https://files.pythonhosted.org/packages/b4/90/e2159492b5426be0c1fef7acba807a03511f97c5f86b3caeda6ad92351a7/psutil-7.2.2-cp37-abi3-win_amd64.whl", hash = "sha256:eb7e81434c8d223ec4a219b5fc1c47d0417b12be7ea866e24fb5ad6e84b3d988", size = 137737, upload-time = "2026-01-28T18:15:33.849Z" }, + { url = "https://files.pythonhosted.org/packages/8c/c7/7bb2e321574b10df20cbde462a94e2b71d05f9bbda251ef27d104668306a/psutil-7.2.2-cp37-abi3-win_arm64.whl", hash = "sha256:8c233660f575a5a89e6d4cb65d9f938126312bca76d8fe087b947b3a1aaac9ee", size = 134617, upload-time = "2026-01-28T18:15:36.514Z" }, +] + +[[package]] +name = "py-cpuinfo" +version = "9.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/37/a8/d832f7293ebb21690860d2e01d8115e5ff6f2ae8bbdc953f0eb0fa4bd2c7/py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690", size = 104716, upload-time = "2022-10-25T20:38:06.303Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/a9/023730ba63db1e494a271cb018dcd361bd2c917ba7004c3e49d5daf795a2/py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5", size = 22335, upload-time = "2022-10-25T20:38:27.636Z" }, +] + +[[package]] +name = "py3nvml" +version = "0.2.7" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "xmltodict" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/10/7e/fa282e456b87570d663ce97946b4dcb16850d4495ce4bd625a1a10c8ed56/py3nvml-0.2.7.tar.gz", hash = "sha256:09ee1d04598a6e664e24465f804ce3bfe119a6fdb5362df1c168f8aa929fbd73", size = 58224, upload-time = "2021-11-22T14:30:27.541Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/3a/ea6f2419bd20f97f65ee55a9910c722313fe99cacc0bf77afb4b74b446ff/py3nvml-0.2.7-py3-none-any.whl", hash = "sha256:30101170d1f51419c8d21fd8ca6cdc333a552b4f8a945c2fc7d107d77e4220dd", size = 55503, upload-time = "2021-11-22T14:30:25.794Z" }, +] + [[package]] name = "pycparser" version = "3.0" @@ -945,6 +1060,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, ] +[[package]] +name = "pyreadline3" +version = "3.5.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b6/6d/f94028646d7bbe6d9d873c47ee7c246f2d29129d253f0d96cb6fcab70733/pyreadline3-3.5.6.tar.gz", hash = "sha256:61e53218b99656091ddb077df9e71f25850e72e030b6183b39c9b7e6e4f4a9bf", size = 100368, upload-time = "2026-05-14T17:55:04.471Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f7/5e/35c856e186b74678c24927847ad9895a51f1bc02a0c6126477a6c6040064/pyreadline3-3.5.6-py3-none-any.whl", hash = "sha256:8449b734232e42a5dcd74048e39b60db2839a4c38cf3ae2bf7707d58b5389c0d", size = 85243, upload-time = "2026-05-14T17:55:03.262Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -1282,3 +1406,12 @@ sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac8 wheels = [ { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, ] + +[[package]] +name = "xmltodict" +version = "1.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/19/70/80f3b7c10d2630aa66414bf23d210386700aa390547278c789afa994fd7e/xmltodict-1.0.4.tar.gz", hash = "sha256:6d94c9f834dd9e44514162799d344d815a3a4faec913717a9ecbfa5be1bb8e61", size = 26124, upload-time = "2026-02-22T02:21:22.074Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/34/98a2f52245f4d47be93b580dae5f9861ef58977d73a79eb47c58f1ad1f3a/xmltodict-1.0.4-py3-none-any.whl", hash = "sha256:a4a00d300b0e1c59fc2bfccb53d7b2e88c32f200df138a0dd2229f842497026a", size = 13580, upload-time = "2026-02-22T02:21:21.039Z" }, +]