From 8b61529942cc0ffd41293edacaade40450003712 Mon Sep 17 00:00:00 2001 From: Michael Xie Date: Wed, 15 Jul 2026 01:39:24 -0700 Subject: [PATCH 01/55] add/update dependencies for bandsilo --- pyproject.toml | 12 ++++++++++++ src/giant_python/assets/psfs/dil-05.tif | Bin 0 -> 1008 bytes src/giant_python/assets/psfs/dil-07.tif | Bin 0 -> 1008 bytes src/giant_python/assets/psfs/dil-09.tif | Bin 0 -> 1320 bytes src/giant_python/assets/psfs/dil-11.tif | Bin 0 -> 1320 bytes src/giant_python/assets/psfs/dil-13.tif | Bin 0 -> 1320 bytes src/giant_python/assets/psfs/dil-15.tif | Bin 0 -> 1632 bytes src/giant_python/assets/psfs/dil-17.tif | Bin 0 -> 1632 bytes src/giant_python/assets/psfs/dil-19.tif | Bin 0 -> 1632 bytes src/giant_python/assets/psfs/dil-21.tif | Bin 0 -> 1944 bytes src/giant_python/assets/psfs/dil-23.tif | Bin 0 -> 1944 bytes src/giant_python/assets/psfs/dil-25.tif | Bin 0 -> 1944 bytes src/giant_python/assets/psfs/dil-27.tif | Bin 0 -> 2256 bytes src/giant_python/assets/psfs/dil-29.tif | Bin 0 -> 2256 bytes src/giant_python/assets/psfs/dil-31.tif | Bin 0 -> 2256 bytes src/giant_python/assets/psfs/dil-41.tif | Bin 0 -> 3192 bytes src/giant_python/assets/psfs/dil-51.tif | Bin 0 -> 3504 bytes 17 files changed, 12 insertions(+) create mode 100644 src/giant_python/assets/psfs/dil-05.tif create mode 100644 src/giant_python/assets/psfs/dil-07.tif create mode 100644 src/giant_python/assets/psfs/dil-09.tif create mode 100644 src/giant_python/assets/psfs/dil-11.tif create mode 100644 src/giant_python/assets/psfs/dil-13.tif create mode 100644 src/giant_python/assets/psfs/dil-15.tif create mode 100644 src/giant_python/assets/psfs/dil-17.tif create mode 100644 src/giant_python/assets/psfs/dil-19.tif create mode 100644 src/giant_python/assets/psfs/dil-21.tif create mode 100644 src/giant_python/assets/psfs/dil-23.tif create mode 100644 src/giant_python/assets/psfs/dil-25.tif create mode 100644 src/giant_python/assets/psfs/dil-27.tif create mode 100644 src/giant_python/assets/psfs/dil-29.tif create mode 100644 src/giant_python/assets/psfs/dil-31.tif create mode 100644 src/giant_python/assets/psfs/dil-41.tif create mode 100644 src/giant_python/assets/psfs/dil-51.tif diff --git a/pyproject.toml b/pyproject.toml index 29877a2..7fc2efd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,9 @@ classifiers = [ readme = "README.md" dynamic = ["version"] +# NOTE: The SLAP2 integration (BandSILo) source-extraction backend requires +# ``slap2_utils`` (raw SLAP2 data-file reader). It is not published on PyPI, so +# it is installed directly from source via the git URL below. dependencies = [ "numpy", "scipy", @@ -23,6 +26,10 @@ dependencies = [ "h5py", "tifffile", "joblib", + "torch", + "opencv-python", + "pandas", + "slap2_utils @ git+https://github.com/m-xie/SLAP2_Utils.git@multidatafiles", ] [project.optional-dependencies] @@ -45,6 +52,11 @@ dev = [ [tool.setuptools.packages.find] where = ["src"] +[tool.setuptools.package-data] +# Bundle the default point-spread-function template used by the BandSILo +# (integration) source-extraction backend. +giant_python = ["assets/psfs/*.tif"] + [tool.setuptools.dynamic] version = {attr = "giant_python.__version__"} diff --git a/src/giant_python/assets/psfs/dil-05.tif b/src/giant_python/assets/psfs/dil-05.tif new file mode 100644 index 0000000000000000000000000000000000000000..cfb47a1c8e74d729435ff44fe9204f47b52741a9 GIT binary patch literal 1008 zcmebD)MnsdU|`t71SEh!J3+xlCu)K9%yrXkV%|)$jy%mtJ+iXtuI#jFx-Mr^@Uq=Tja}NN{rMSd5Dnsk#6j{Pbs+VZcZ%BxEi<$^ zQ|N3H{YB4)`G&4dlzFj@*yPz@8pH>QgXBT#Ka9hc7Fjbsj<$*XZ*03p z^nvYrNj^Ie4dR2uLGmDVAoag~^;^GRy36{<6mgpmHc_@Lrloe@|NpRK&K0r;(I7rZ z93&4?2U35#W{35JI4&E@uX}7>K9#k5Y@=nrW!_o)tsj^jKs1OC5(mkH)PdB4!tUSS zBQ`N7-`MQ0^RYX=Gu7V3Nzvh-WP$^T2Ju1SAbF5Fka}|LA;&-Dgc~{WLryv%C%y55 z0uKoOF|aT&LYN4W5h2D46=lRC#>B`B*Q)?k$BZNfGnWmEm>{D7T(1CBuP7FEVp!}I nN7B0ln&+e#S>fh6%wYhtrLl;~V6j63NsJllmS8MmP$>ogvWqGW literal 0 HcmV?d00001 diff --git a/src/giant_python/assets/psfs/dil-07.tif b/src/giant_python/assets/psfs/dil-07.tif new file mode 100644 index 0000000000000000000000000000000000000000..5d4c8a284f55175e2baf09f338f22312a81cace8 GIT binary patch literal 1008 zcmebD)MnsdU|`t71SEi<{M}?5JIU2HN2E4bZ?|W+d027QDulhm`Y%U_4TuKuLE<2J zkUEh1zFjpoTzjQzF}_!1bGqb`&8=H+!8C{u5(mkH)PdAXOu1rx@ok;W z)d>QgXBT#K-NMbN^*|3NSG77-;3PAOWVo@iC#a?kF my-T2ZPKuEgZl1#&1~6M1iRXH}s zz9BY&zYA=BDC^iTOjfdqePR!$L41%nNFJmPq<-%GNSk0M6&ruM1J?Qvd2JYGx7bYi z@W6)i+*&XV;)BFN@*s5}^_&Y$Z0NnoHZ1eWnO&g}LEZa@(`F2kq zr`YG5eQ$riM%)2JgZLnEkUU5oNd29X)i%5Rxom^K%(mqSI%%ibxXFHPwWY)Ve>n~y z8pH>QgXBT#KMH4w46{1F0v+ z9&-FcPPma1KjfqXa?%?)>6)DULry;Bhh%Yve+(=Pj8Hb5Vnm4XLIoL-#Dt+@OpMHM zy$Vngrx90o938jF|=7CSVM P#GuA81Y;3{I+Fnao(8^Z literal 0 HcmV?d00001 diff --git a/src/giant_python/assets/psfs/dil-11.tif b/src/giant_python/assets/psfs/dil-11.tif new file mode 100644 index 0000000000000000000000000000000000000000..76a1b0d1936de0c269a79aad91990332a8d0f53b GIT binary patch literal 1320 zcmbW1O-NKx9L4W*(mwo1i5#7TL=hw-A_>&_-;b4HPF6Egw#Y>&f)$iXf<@FqF@u7E z3?--1B)@7QZk#s3*h0_=S|mI~ji5~`n8Dx#l4p92vv{jJi+A2R=XdXYGjs2&sw!do z7-OrBoy*Rrvnx!ehdl4vpyDxA{Fw;8Jc;x88fmiiT z>M1+U3+e*=?#oHxe>Nzu`A({Do~sH9j~qQX@T*Htc*YeMH`9VV9GMYqH^-#!7%5qO zM?v9{qX!3GXC0mIqw~Az{69MTKxe<{>}#F#qjOFV{yXAqo9$tSbsraFH%qrdBOyCv zWl4r3J~rRV90}R~c~fk%EF&{MHq#n=*oMotS=X5`cF}rQxs2raeCh>@-5;^Z@@&>9 PPROk3n9nA&9+UkAL!`b| literal 0 HcmV?d00001 diff --git a/src/giant_python/assets/psfs/dil-13.tif b/src/giant_python/assets/psfs/dil-13.tif new file mode 100644 index 0000000000000000000000000000000000000000..5f9923e95f2573bd0e5cfaeb1a91343261298f13 GIT binary patch literal 1320 zcmbW1Ur19?9LLXXD{Po9H~*lrW-vLFaJWts z2qBhS>zDP<`0ypVo~oyk?%UKAJ4dSR87)L^(SyXR1}Hpo^x(jI%F}fI^kWLn?4<*9 zDf(7(pZc12@gq+IL*bF52M7LQtdx6#FDW|ogwjO=hvZ7Sis}aSeq>jvgHNO#C+m z&K}{>^i@8!l9azj2UPJyjefU$QbXa9qX!3GHO6?l^$fqw4sv_%J$bNtShXw#^@Yi! z8VZjbJvi_t>zI5Wli$tc|1sGICi~4~Uz?mClXF`9_lS#C;TD#CpA+kUw#^Q$yv%23 zn=DsuY|zeJdD*{tw>V_mEpKkD*B)Eyz?C_yTb?)ell`u$v^MAF(|5(jeU(F2?Xbq~ Pyv&|XoN&nO#}t1648^B2 literal 0 HcmV?d00001 diff --git a/src/giant_python/assets/psfs/dil-15.tif b/src/giant_python/assets/psfs/dil-15.tif new file mode 100644 index 0000000000000000000000000000000000000000..7803a7ea404c65a5591d9460fca279c50aeb77ce GIT binary patch literal 1632 zcmbW1T}V@59LCS_II*E@hAwPYBeYL4(hEV~|9jqwETe+7i<;7f*@w|ZU6>NsSVrxk|`BhJ`Mlby~aF)w_#5=Xsvrd)|G#=U-N~ zRY_G8CD*k4w)_fz`@$+4=GpDODfYHD#C{iON29rCKiLg> z$L{0=S@o=+nMa4%)cc1~g-4Da9C$~6h_wg8thM<&%W%D9drQ5nGvHw_Hu|Cpj~qQX z@ZQqRwBWGPVBtsR>>Fo#_6n+<&7>>#wNZsfjvgF%k8L-t8A+v+6VKW4kcl2O-lB_v zOg=nNz@hNS(Srj&95GWcWtugmPqV51Zt9&b=Qjt&xwl2rq43Djg9C4#G0}C)4Exo) zfsS;o=kBoxpWM=*AA8lML*bF52M7LgzKzPyFR^;Bn>r_&`Khr=eSR{iZ{L|Npzz4i zg9BgaO(ws29r<#)>A>rM>o1QpZz%DXUE{I+rIt>^Ur;@W|1F13!1Sg!1cZ z=*`g!G+2F&N32bHXO&&JX7&pxJaY8lz}K%Ns<_`ur*j`r!{e*Gs-scAS7Q;LZXuxX z$kBrXFSC!#_mTPCWd0wSb0BlxWX`qB^CR<|%FF|qxg#^*EPqF%vZN#{s&O3?^?#Ob zglb%tZDdKR7F+8uGA%Cqci)r*S%zwjt+g6!vl4K&1pBUvTbpUj_|>W@c0bXsM6c}$ UvYZ5aY>LY?$G-puFfv@oacFd?|Jv}p1;dg z&lPhVS0^mKExy9B(`1V~XrE&l+1J<5t${g_x;n^zK15J>rPLOa z424II9vt|IJp%21YN3yZn#tF+T?!m4lJ{QjmKSVyateBDS7@^{{posn9`4>vL>JaY8lz<0NvB+hVwCg|juM_cB7h zA3>?Z?UUc^G_n4;$e{4Z(Srkj_~0;&w%N&)eMnKEU(#8;^8KuyIo57vP!ZyGNt1F0K7;uegFUf literal 0 HcmV?d00001 diff --git a/src/giant_python/assets/psfs/dil-19.tif b/src/giant_python/assets/psfs/dil-19.tif new file mode 100644 index 0000000000000000000000000000000000000000..75630d6532e51a7c97d63f8d1159ff3a8171c192 GIT binary patch literal 1632 zcmbW1Ur19?9LMi;YDUW~S93WlY))x`6N7B+ch0G$Ht|6yXcZ+yFJfjBu^tQ}ghFhz zjg1jKjM^AM+Q{YAVuWUNdS<`vU{HAE=)r-H z^;S}%{wO^PoS^BFkJS3Ai|Ll9+0SSphr%OA4-R~Vw~T5A+iA+$PZe*SY~k|?yT7iJ zKPw1uC_HlX;J{D3=%IeOiw@XEXkHF7x3!s%^-l8IcS|XSM~)sG_#yLUGVG|QPi%sY zEIegfT7vvyfnJ0TRtqRRa`fQ9j~Cg=6Y3+IVU)aWNw&Q8Emcqg-4Da9C*9!Dm{GDK^x9QD3~+IzR!fYLoO4p zU|+C_HlX;J`=ZIF0R!Qs?X>g=RynV~?L#HJOCo?-o#a6B2Dk(re& zN0XagyG_Y*GqQj8&CikT7uIIHNl0NRK9G8nuX>5=bnU2otb7})}VxjBC9V_B)u7_Ddu9U zNK!!y`ru32i_=PpFm)&?LYYX60>MD7NTT|p`K;U8hkd&Hu;)I{^E>zK<=k^JGqr-R zAP7d)>f7pTzWP4Z74_-0PNzQfV=Sc>9-!i`deT`eE`>*q9vt}X-69PJoFbztkKSvI z6g*cz*V5HA)wJHF@W|1F13#R7nDR$6NYj!@i{%M4{Vb9kU*hScd8bR^k)sC(-omt$ zZsn9}*h06=(KJ+Lq?MKnG-)~OQh4O(!GZVPZ6b$XF&X00=*0Oeq`CQl^t+;1ttp>D z;gO>U2j0G;f-2wbr?h~*^g8J+SuI!Dk&bWdbh5t$g-4Da9QZR+TglgIqI0j>=ult* z>z7uT?N^x;Ro);$;gO>U2R^}GNz+MMTH5BInuWWJ47pNi)*H#5D{?42a`fQ9>+A-4 zI9f=LCWk3;s)xnxu}UR3{P^|D(Hsho96dPjWd|$in7W37W@o4*Gu;gugcSLC_HlX;J^>) zdT9A;7j@rxN<%rVY%-=^3SSK1W1kW^6dpNxaNsqcA5%qJKmE9*VlSGXGyhVn)Hol+ zM}mn%;gO>U2R`{?HPwtY(?s10#oGGWoUKu6bwqG;d=7`gBS#Moypnm8d>WfIpC6J}g0BLyL+f@0ABxd$y%lc32gL=>`KG>aBVvKmFj8TX|u zCvEXTGH6o=L$i`B3#7#|Un--B5(~{rh_uOXx=r&iPj?>XyWjWy&ONi7dro3vq_9vB zgmm}$xA~W;K95wh25N0gro6dAD$jl%Q>|J~8*_qf3XdE;IPi@-4$w@7kxUgiujF zr6-Ffc5bFkO=g;!`bb?KG3>{qTn2?ljvgHNqZT6-_cc&0t0Jv5N#S#KY)$YS^Dop$ zPRHZRWgE!4Kg-4Da9QZwAEscj2lJ&_8TH)Em zBHwM5!V(`#+7YWw;gO>U2VMwoBew^qsq)S&8OnQ^dazpZ9QNYl0joI_9yxk&;B(ie z(T9#cGK5Z3^VkUMFFG&v#%g#-Wi*GvBS#Moe5}5hYA&24j}5QrYU)F_J<}{%Bh>uZ zSH_|6$kBrXf8F&yWn0hDiIfpqrs`n933XB*_vgjh^&AS196dPj*61%3cke6>l#G(F z_X?BBN+n~z4>xF|I20Z^dT`+1<~^gV_zwE1o1h~}H<;i1DyiG#!}IfX914#dJvi{D zi|wQrZbnVt+-nmc5q43Djg9Bevbez=F7E-+&rGtmsnHY6OvSbJG zl)yL+g-4Da9Qc`@aY}mHKwT@pQ9(&33mZBmd9Bm%*A?*`3XdE;IPgm5QSyD1{BBDA zA0_KR$$C?=u9ZALN}f|C`+<^uN6G%CWFJ+sUn@BmeE%LP!f(M<5anyTi2pNhITRhT zP&r#5y4c4ql`|KI?B9Leon(tdKl@led2E0aZi$n510BXD%V(onbhF=&Hw(6Fjgu_M S$s7?5nTz~cwmHeZi1(F$4Ghm>X?lBH>>nPCPI7@A~}nPn;YthIKrt9KWB-sgFK=RNy4?>Pwx>m@Hq zlG0QQZws%k(E(ERl~Y9Fx!C82TVvanrqb~?Ze% z(SA)SWepZniy_~x@W|1F13%VKO~;07$nSMErS7y+{n#X(J#S#OIVJ{$M~)sG_#Kua zIxyTsVHsAk?S4QX_MKpwXLHQM63U_Q$kBrX|I2cjGIPyzck~v;8S~iMObw53F64$Q z7wig;96dPjVTvXy?Wv-owrT1~y}>s1XYuXhLp=QLoL%9OqX!3GomNHT4?E~yh9A?1 zK4rEIl{~m^u_&pD5KwsJ=)r+s)^DQ0ggW}B@nq4{FWA?>i+rY9EBtL-K;e<22M0cR z)k~_2vr&o|Cf&zYW(>0O>_;m_L5L7gc;x88ffq5|bZz5jiu&@7h6Y<1?>2K?Wq>$i z*eam#$kBrX|H-R^Cf2>8-!U^(ezS?CL|gd8NnfG=86}|b$kBrXpQ7(1rRF;6lV&L~ zsDVYNmvGDNCE{w?8UckzjvgHNoPsB$+ciRwKC=|>{g@e=D|nPCKpbh@ETHhn(SrlO zI`%saoa&;3mnNxkc?)}^E#sy>ji{ZE6Hs{M=)r;S&F!Tvo$WNx@5Sy9wX;Jb6+F*F zD|`eAC_HlX;K0{sP0^jHDKchI5 S+{_W-k|`aZCD~2psH8t~^T_1@ literal 0 HcmV?d00001 diff --git a/src/giant_python/assets/psfs/dil-27.tif b/src/giant_python/assets/psfs/dil-27.tif new file mode 100644 index 0000000000000000000000000000000000000000..f95c50a53a8f354161ffb348ee2baf9c6b648a4b GIT binary patch literal 2256 zcmbW3ZA^_}9LAqh(XrGiDW^_`Mm24T=H)~G|8uKZvW&x8W6g|Fqg69sytGnju~8`z z$>gOM6j_?3TBU52HdAJ5s5#6!A=WWWbR9?fa6aAV!?~X8x__Ldr=uyg-4Da9QcHf zujsmOEA^$;Qb6GuNTE8{#HE64e<$2=7Zj~qQX@N*m6sc>BjecfMA z4|U z2VPg1K_$9=8hiJd4t;;lDT~jle*3gy!NC9lg-4Da9QYkwj%IA&y=hc;HL z7PDToj0pjSM~)sG`1bG#3QE$k#puOPUccecq;j<)#2}JSM+hi9a`fQ9e{t@or;}s# z%^LQ1Z{@+-Vs%57nD4<=m)AzvCxI$DlU6>M%Cwe3m50r0t$~D zJvi`nQF)ZNw2?OVXgN2mk}n)RrN)-H3DdU~0t$~DJvi`f9gEoYfb3c~wG*n(yH)az?`i6dpNxaNr-F&!)qB z2dOF7Oc90-_BWQQ=|u+7T(nj|;gO>U2i||SiH4la6!%%nmi$g`87Nh6{Pq)JNl^j{ zj~qQX@Ww<9|48hn7E=%>*LQL1rc3INd~Y$35iX$c$kBrXFLNE4?<4cO$^1Vu^FU_4 z$;@k+`$y(Jm01sD)*YGkO=caHS+8aG1)2RvW}lPUPj!E{Vr4>cQZ%;XDbf7TTy3Fd zTDHWNIcOZG#xAsFj?=P#=bdRM^U%0Yjdiz;^|Hfx+qv$%X=Af(&lJ9z8B^yIrHb{~ WU?=mlbB&;BnWOEqY_*fwD&-GB&sTc@ literal 0 HcmV?d00001 diff --git a/src/giant_python/assets/psfs/dil-29.tif b/src/giant_python/assets/psfs/dil-29.tif new file mode 100644 index 0000000000000000000000000000000000000000..c40ebc4e8deceb98c48826aa5a0476b5b38949fd GIT binary patch literal 2256 zcmbW3drXaC9LL}HoRbteIiVCPG+D!pL*(~+o-uJ+l1pjKXxK1JW^Ob0OID1OOG{*^ z#IRPAE~li6Sp1>fFKx`sWh`3O+`h+={y2X<=a2J!zu)iYdEaw+-sgF3wlJ-`rfKQ! zf8PGQ(oT=ydg~Hux)V1D2WQ~C~6~3jynEeigM~)sG_*J8m>2=db8l752Q!8sJ zL2IDsGZ$&+_!@`8BS#MoeB-KJG$G|B4R1L{CEEVJBr?^%C- zT6&YCDpm`>#z$h_2u}%xM~)sG_;nqRX?j6BooWr>;rs7$c67Rkyw^=eMGuouc;x88 zfj`+{CtI@zH%}YI+4rAwK+7eOn&d0TEf1Gac;x88f&bF*h;~0Pa7>GpukZWH$&>Si zDZ@`DE+h$sM~)sGcwymH-X~AFVa_NC zg-4Da9QgLQUsS%xou6$naaKw$m$s&e&=M~hXCE)2@W|1F1OKC{g1qy5Il|qCwQZ#w zn}1x~GWg2LS0W@79yxk&;8(`g(vj*nba{arKP*4Q*r*WO>Y4^By$6vlLvq>mCa`fQ97hN>6IaB8lvp>(Zlyh!gnrJSt$P@7q z5(UMdNHgS(G$ELgA642M1o|Ix62s<#$v0e^lmy z%6wCq*DCjq%6+P`9;mE4D(jodI;yf>tLzIZ`;W>#r?Q`#|8B+FPtBz1&f`we|7Tv# zQ16n3I5UH8>>L~9%#2;Kf9LJyBI~VtcaHUTj_vD$>*wOSzFo%daXwS{>u#OrlUbVM XIKV|Vz{NE}yJSY^XPM(7b5`1KxJ@=A literal 0 HcmV?d00001 diff --git a/src/giant_python/assets/psfs/dil-31.tif b/src/giant_python/assets/psfs/dil-31.tif new file mode 100644 index 0000000000000000000000000000000000000000..bf617a3ba87252a86ac05038d3298b0b2b6474bc GIT binary patch literal 2256 zcmbW3TS!zv9LCRjSxs1Vsa$|&uWnwNbJ;!t?x=)r+^XuZpl!`!GWNJT~I`zZX&3T`-@ z!?OpanpDeU?Xon)n4FBXf;aweeg$kBrXKQv~h%r|bd%~MM|dOB%i!zHe- z(TQs7Y5|2ujvgHNntXREt#_gLI$t{B(nPxlPjkyocab<6DxmPl(Srkjr*NDdnQ^4Y zwMJ@M)=am8j`OT~SMl>B7fWjk34-Wi?+yNGL&z&skM!KPGq70VHjSqFg6th7<;gO>U2fp-?n$AV~ zleg7GQGG3BN-5yMqk0i}W|M%zBS#Mo{H|yXd2bJ-kPtJCWLfEbY(DQQH;6-9)(a>+ za`fQ9k0o7a!~Qprm1)IEQQ3~?MD;(@*h1C3 zY`HCSP&v+xU24l5=VkxStG1JARi1NWJ#Axkb~wGA>w3)_n`L{ZFsPj8&L@hL+2bX4 UGNYYq1kcMHZJ#C1PG+l=-(O)nr2qf` literal 0 HcmV?d00001 diff --git a/src/giant_python/assets/psfs/dil-41.tif b/src/giant_python/assets/psfs/dil-41.tif new file mode 100644 index 0000000000000000000000000000000000000000..2f42c4a86ed3c85f69eb35efcaf9273edc310947 GIT binary patch literal 3192 zcmbW3e@xVM9LDcBz#oKg7U*#hB%;W#e(5R5J)aMShlltBhz5a%V=I+_k&qBzuqFc$ zQ=>7bzzG6PU9;NM3}ul0vM6ofw6$^zr3hz=Q?X**b6&(B{Pnp%?z!iAUf=tE@9uk_ z&)aGZ*G$)FG_fA`C;KDr#Bp=&umypqV^I5aGnOpdhT^Mt%*C};4y9-A=dvfG$WhNNb1)}4+x1k7#7Tb4hg4ULX$oQ*2}YSF~i@X^vv0lBYz+*6_z8exa+YPMMq88 z=;e(yZ@MDn{zIi_&Ym3k`8m&Hu1?~5&WqVL;9`{Q2Fg>~uwc|E5_)$C6m~k4o;iDR}AjhiF>$L~D#V*27dQ7y*InuHyT2kqmvnNMBaZx%(_vd48=w8^0 zW{QfR4hxIBQ8wpBODa8c_TjXxhPEZ6jwrQqQ*8`ezDdpsr1a*lOsPCos0X`>yi1r{d-B%iSjM2qAJNC zv$B^;Dm`=dQ3nghx%g7+7hPv6WI%>6x=9N8TE+3_Yc(_%x>u zk>^~5^_8O{qCZ5I>{}_R^vv0lBfs{eENl+RN5!>z%+TmW`0ZBlP5%?JcO*_y>6x=9 zNB%}tn*Bb?$8=K*4qlus9u&SWjvw=rJ1Zfn^vv0lBcFLZ7z63KXsxNnANK1u-D(y2 z-v!9NZ{sADo;iDRfEdgkoOk?-k#1FN@I zA?=q2Y%CbTP;FG$s|SyJhlvnNMhUDQ~axrDXAnfnwli%|7gk#V@^0tHjXn|Udo!rTX@Y&B1j^x ztRMlqiy(v`C|<~sQK-#ftbq{|9mnw2XbqMPnip`qkZ0S74}9s_4}12U^LzJQ*ms}j z|0X0v8N3Y!Lz3_Fr{|ATpM5Fbk@mRAF&`k&RDh304@8Tj5W^Olgi6nxJvs7qQ?1BW zsaU#c8QPa-V}y?t<&Hi0#Nl))J#+Tt$j@Ip3%R4VAZ62HEX!Ys{6;Izq?W>btHq`C z%-NG8Z(rcR$Qg67eQi1-m+i!gMmvUPRAKPb?_5gHoIN@6vE^G~ZTcMVueZbS%QmDh zNyjsfo48wl&876r*^?t5o3b5i?|cDsVlsyASOve7CunR-l zdgkoOk)IH-93{JLcwB44b4D#o?c!9R9@c%IG@YnG>)6V+c)>6x=9N51UgySSa;z^VDiaMs*go{Y7~UjZHR zVA=phrDx8b9Qlctcfx$G7)9r5u`arwcy7rPOO}`F7#F6f^vv0lBX7wlMtr6PNg?0j z*V;fCJv~c0$KREoLwyyMo;iDRFWKRVZe&h@5qUF-aQbbhBg>w(U?qqDy0tfM;Xwa&hv zv;XMqb2|H}&c3ZPA8@YtzjKJryrVN$>CA6BbE3{Xsx$ZM%-1?|yw3fC|39}HhGzy( zgVFuEON{?ze(uoNEsJw!FBm<#jveIAJi2B7zHhG{vOr@%*RcWaV}pC(Og)^}-0j$f k?t7*%qgU7aDZ9b-I=qLhe-Gyv-YxTRf0lQ8$lR6TZ)K8kp#T5? literal 0 HcmV?d00001 From ac1025a0b4f643f41dda89df99a175a6b793b483 Mon Sep 17 00:00:00 2001 From: Michael Xie Date: Thu, 16 Jul 2026 00:14:07 -0700 Subject: [PATCH 02/55] implemented h5 loaders, basic dmd geometry functions, and parameters --- src/giant_python/bandsilo/__init__.py | 13 + src/giant_python/bandsilo/geometry.py | 367 ++++++++++++++++++++++++++ src/giant_python/bandsilo/hdf5.py | 334 +++++++++++++++++++++++ src/giant_python/models/params.py | 50 +++- tests/test_bandsilo_geometry.py | 176 ++++++++++++ tests/test_bandsilo_hdf5.py | 253 ++++++++++++++++++ 6 files changed, 1192 insertions(+), 1 deletion(-) create mode 100644 src/giant_python/bandsilo/__init__.py create mode 100644 src/giant_python/bandsilo/geometry.py create mode 100644 src/giant_python/bandsilo/hdf5.py create mode 100644 tests/test_bandsilo_geometry.py create mode 100644 tests/test_bandsilo_hdf5.py diff --git a/src/giant_python/bandsilo/__init__.py b/src/giant_python/bandsilo/__init__.py new file mode 100644 index 0000000..6602879 --- /dev/null +++ b/src/giant_python/bandsilo/__init__.py @@ -0,0 +1,13 @@ +"""Self-contained BandSILo (SLAP2 integration/band-scan) backend. + +This package owns everything the integration source-extraction backend needs +(HDF5 IO, geometry/PSF, trial-data reading, background/rho, peak detection, +NMF localization, trace extraction, and orchestration), ported faithfully from +``extractSLAP2IntegrationSources.py`` in ophys-slap2-analysis. + +It is deliberately self-contained: shared-looking kernels are duplicated here +rather than promoted to the generic ``math``/``io`` homes, so a future standard +(pixel-movie) SILo backend can be added without depending on this package. The +public entry points live in :mod:`giant_python.pipeline.extract`, which wraps +:mod:`giant_python.bandsilo.pipeline`. +""" diff --git a/src/giant_python/bandsilo/geometry.py b/src/giant_python/bandsilo/geometry.py new file mode 100644 index 0000000..c01c753 --- /dev/null +++ b/src/giant_python/bandsilo/geometry.py @@ -0,0 +1,367 @@ +"""DMD geometry, reference-stack/PSF loading, and sparse projection matrices. + +Ported from the setup blocks of ``extractSLAP2IntegrationSources.py``: +``ref_pixs_to_drc`` (flat pixel -> depth/column/row), the band-registration +lookup-table reader, ``subsampleMatrixInds`` construction, reference-stack +loading, PSF loading/cropping (from bundled ``assets/psfs/dil-NN.tif``), and +the sparse ``H`` PSF-convolution matrix used to project image space into +superpixel space. +""" + +from __future__ import annotations + +from importlib import resources +from pathlib import Path +from typing import Optional, Tuple, Union + +import numpy as np +import tifffile + +from .hdf5 import load_struct_from_h5 + + +def ref_pixs_to_drc( + ref_pixs: np.ndarray, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, +) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + """Map flat reference-pixel indices to DMD (depth, column, row) indices. + + Parameters + ---------- + ref_pixs : ndarray of int + Flat reference-pixel indices. + dmd_pixels_per_column, dmd_pixels_per_row : int + DMD geometry. + + Returns + ------- + ref_d, ref_c, ref_r : ndarray of int32 + Depth, column, and row indices. + """ + ref_pixs = np.asarray(ref_pixs, dtype=np.int64) + plane = int(dmd_pixels_per_column) * int(dmd_pixels_per_row) + npc = int(dmd_pixels_per_column) + ref_d = np.floor_divide(ref_pixs, plane).astype(np.int32) + ref_c = np.floor_divide( + ref_pixs - ref_d.astype(np.int64) * plane, npc + ).astype(np.int32) + ref_r = np.mod(ref_pixs, npc).astype(np.int32) + return ref_d, ref_c, ref_r + + +def load_lookup_table(path: Union[str, Path], n_dmds: int) -> dict: + """Load the band-registration lookup table (``bandRegLookupTable.h5``). + + Parameters + ---------- + path : str or Path + Path to the lookup-table HDF5 file. + n_dmds : int + Number of DMD paths. + + Returns + ------- + dict + ``{"allSuperPixelIDs": {...}, "sparseMaskInds": {...}, + "fastZ2RefZ": {...}}``, each an inner dict keyed ``DMD{N}``. + """ + lt = load_struct_from_h5(path) + all_super_pixel_ids = {} + sparse_mask_inds = {} + fastz_to_refz = {} + for d in range(n_dmds): + p = lt.get(f"Path{d + 1}", lt.get(f"DMD{d + 1}")) + key = f"DMD{d + 1}" + all_super_pixel_ids[key] = ( + np.asarray(p["allSuperPixelIDs"]).reshape(-1, 1).astype(np.int32) + ) + sparse_mask_inds[key] = np.asarray(p["sparseMaskInds"]).astype( + np.int32 + ) + fastz_to_refz[key] = ( + np.asarray(p["fastZ2RefZ"]).reshape(-1, 1).astype(np.int32) + ) + return { + "allSuperPixelIDs": all_super_pixel_ids, + "sparseMaskInds": sparse_mask_inds, + "fastZ2RefZ": fastz_to_refz, + } + + +def build_subsample_matrix_inds( + all_super_pixel_ids: np.ndarray, + sparse_mask_inds: np.ndarray, +) -> np.ndarray: + """Build the superpixel -> reference-pixel index map for one DMD. + + For each superpixel, picks a reference open pixel: if the number of open + pixels is odd, the median pixel value is used; if even, the middle element + (by sparse-mask order) is used. + + Parameters + ---------- + all_super_pixel_ids : ndarray of shape (n_superpixels, 1) + Superpixel ids for this DMD. + sparse_mask_inds : ndarray of shape (N, 2) + ``[open_pixel (1-based), superpixel_id (1-based)]`` rows. + + Returns + ------- + ndarray of shape (n_superpixels, 2), int32 + ``[ref_pixel (0-based), superpixel_id (1-based)]`` per superpixel. + """ + num_super_pixels = all_super_pixel_ids.shape[0] + out = np.zeros((num_super_pixels, 2), dtype=np.int32) + for sp in range(num_super_pixels): + inds = np.where(sparse_mask_inds[:, 1] == sp + 1)[0] + open_pixs = sparse_mask_inds[inds, 0] - 1 + if len(open_pixs) % 2 == 1: + ref_pix = int(np.median(open_pixs)) + else: + ref_pix = open_pixs[int(np.floor(len(open_pixs) / 2))] + out[sp, 0] = ref_pix + out[sp, 1] = sp + 1 + return out + + +def _ref_stack_group(ref_stack: dict, dmd_ix: int): + """Return the ref_stack subgroup for a DMD (``Path{N}`` or ``DMD{N}``).""" + return ref_stack.get( + f"Path{dmd_ix + 1}", ref_stack.get(f"DMD{dmd_ix + 1}") + ) + + +def find_reference_file( + datadr: Union[str, Path], dmd_ix: int +) -> Optional[str]: + """Find the REFERENCE tif for a DMD under ``datadr`` (recursive glob). + + Parameters + ---------- + datadr : str or Path + Raw-data directory. + dmd_ix : int + 0-based DMD index. + + Returns + ------- + str or None + Path to the first matching REFERENCE tif, or ``None`` if none found. + """ + patterns = ( + f"**/*DMD{dmd_ix + 1}_CONFIG2-REFERENCE*", + f"**/*DMD{dmd_ix + 1}-REFERENCE*", + ) + for pattern in patterns: + matches = list(Path(datadr).glob(pattern)) + if matches: + return str(matches[0]) + return None + + +def load_reference_stack( + datadr: Union[str, Path], + ref_stack_meta: dict, + dmd_ix: int, +) -> Tuple[Optional[np.ndarray], np.ndarray, Optional[str]]: + """Load the integration reference stack for one DMD. + + Reads the REFERENCE tif from ``datadr`` but takes the channel list from the + trial table's embedded ref_stack metadata. The raw counts are scaled by + ``1/100`` and reshaped to ``[channels, z, y, x]``. + + Parameters + ---------- + datadr : str or Path + Raw-data directory. + ref_stack_meta : dict + The trial table's ``ref_stack`` sub-struct. + dmd_ix : int + 0-based DMD index. + + Returns + ------- + ref_stack : ndarray or None + ``[channels, z, y, x]`` stack, or ``None`` if no REFERENCE file found. + channels : ndarray + Channel indices for this DMD. + ref_file : str or None + The REFERENCE file used, or ``None``. + """ + grp = _ref_stack_group(ref_stack_meta, dmd_ix) + channels = np.asarray(grp["channels"]).reshape(-1) + num_channels = len(channels) + + ref_file = find_reference_file(datadr, dmd_ix) + if ref_file is None: + return None, channels, None + + ref = tifffile.imread(ref_file) / 100 + ref = ref.reshape(-1, num_channels, ref.shape[1], ref.shape[2]) + ref = ref.transpose(1, 0, 2, 3) + return ref, channels, ref_file + + +def default_psf(dilation: int) -> np.ndarray: + """Load a bundled default PSF template (``assets/psfs/dil-NN.tif``). + + Parameters + ---------- + dilation : int + Dilation size selecting ``dil-{dilation:02d}.tif`` (e.g. ``17``). + + Returns + ------- + ndarray of float32 + The PSF template image. + """ + name = f"dil-{int(dilation):02d}.tif" + resource = ( + resources.files("giant_python") + .joinpath("assets") + .joinpath("psfs") + .joinpath(name) + ) + with resources.as_file(resource) as p: + return tifffile.imread(str(p)).astype(np.float32) + + +def _pad_to(psf2d: np.ndarray, height: int, width: int) -> np.ndarray: + """Center-pad a 2-D PSF to ``(height, width)`` with its minimum value.""" + ph, pw = psf2d.shape + pad_r = (height - ph) // 2 + pad_c = (width - pw) // 2 + return np.pad( + psf2d, + ((pad_r, pad_r), (pad_c, pad_c)), + constant_values=np.min(psf2d), + ) + + +def build_combined_psf(psfs: list) -> np.ndarray: + """Stack per-DMD PSFs into a ``(n_dmds, H, W)`` array, center-padded.""" + height = max(p.shape[0] for p in psfs) + width = max(p.shape[1] for p in psfs) + combined = np.zeros((len(psfs), height, width), dtype=np.float32) + for i, p in enumerate(psfs): + combined[i] = _pad_to(p, height, width) + return combined + + +def threshold_and_crop_psf(psf2d: np.ndarray) -> np.ndarray: + """Zero values below ``max * exp(-3)`` and crop boundary zeros. + + Parameters + ---------- + psf2d : ndarray + A single-DMD PSF image. + + Returns + ------- + ndarray of float32 + The thresholded, tightly-cropped PSF. + """ + psf2d = psf2d.astype(np.float32, copy=True) + psf2d[psf2d < np.max(psf2d) * np.exp(-3)] = 0 + non_zero_rows = np.any(psf2d != 0, axis=1) + non_zero_cols = np.any(psf2d != 0, axis=0) + row_start, row_end = np.where(non_zero_rows)[0][[0, -1]] + col_start, col_end = np.where(non_zero_cols)[0][[0, -1]] + row_stop = row_end + 1 + col_stop = col_end + 1 + return psf2d[row_start:row_stop, col_start:col_stop] + + +def load_psf(dilation: int, n_dmds: int) -> dict: + """Load per-DMD PSFs from bundled assets, thresholded and cropped. + + Each DMD uses the same ``dil-NN.tif`` template from + ``assets/psfs/``, center-padded to a common size when shapes differ. + + Parameters + ---------- + dilation : int + Dilation size selecting the bundled PSF template. + n_dmds : int + Number of DMD paths. + + Returns + ------- + dict + ``{"DMD{N}": psf2d}`` thresholded/cropped PSFs. + """ + psfs = [default_psf(dilation) for _ in range(n_dmds)] + psf_combined = build_combined_psf(psfs) + return { + f"DMD{i + 1}": threshold_and_crop_psf(psf_combined[i]) + for i in range(n_dmds) + } + + +def build_sparse_h( + subsample_matrix_inds: np.ndarray, + psf2d: np.ndarray, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, +) -> Tuple[np.ndarray, np.ndarray]: + """Build the sparse ``H`` PSF-convolution matrix (COO index/value arrays). + + ``H`` projects image space (flattened ``z*(rows*cols) + row*cols + col``) + into superpixel space by convolving each superpixel's reference pixel with + the PSF. + + Parameters + ---------- + subsample_matrix_inds : ndarray of shape (n_superpixels, 2) + ``[ref_pixel (0-based), superpixel_id (1-based)]`` per superpixel. + psf2d : ndarray + The (thresholded/cropped) PSF for this DMD. + dmd_pixels_per_column, dmd_pixels_per_row : int + DMD plane geometry. + + Returns + ------- + sparse_h_inds : ndarray of shape (2, nnz), int32 + Row (superpixel) / column (image-pixel) indices of non-zero entries. + sparse_h_vals : ndarray of shape (nnz,), float32 + The corresponding PSF weights. + """ + ref_d, ref_c, ref_r = ref_pixs_to_drc( + subsample_matrix_inds[:, 0], dmd_pixels_per_column, dmd_pixels_per_row + ) + psf2d = np.asarray(psf2d, dtype=np.float32) + psf_h, psf_w = psf2d.shape + filter_size = psf_h * psf_w + plane_size = dmd_pixels_per_column * dmd_pixels_per_row + num_super_pixels = subsample_matrix_inds.shape[0] + + row_offsets = np.arange(psf_h, dtype=np.int32) - (psf_h // 2) + col_offsets = np.arange(psf_w, dtype=np.int32) - (psf_w // 2) + row_offsets_flat = np.broadcast_to( + row_offsets.reshape(-1, 1), (psf_h, psf_w) + ).ravel() + col_offsets_flat = np.broadcast_to( + col_offsets.reshape(1, -1), (psf_h, psf_w) + ).ravel() + psf_vals_flat = psf2d.ravel() + + sparse_h_inds = np.zeros( + (2, num_super_pixels * filter_size), dtype=np.int32 + ) + sparse_h_vals = np.zeros( + (num_super_pixels * filter_size,), dtype=np.float32 + ) + sparse_h_inds[0] = np.repeat(subsample_matrix_inds[:, 1] - 1, filter_size) + for sp in range(num_super_pixels): + start = sp * filter_size + end = (sp + 1) * filter_size + rows = ref_r[sp] + row_offsets_flat + cols = ref_c[sp] + col_offsets_flat + sparse_h_inds[1, start:end] = ( + ref_d[sp] * plane_size + rows * dmd_pixels_per_row + cols + ).astype(np.int32, copy=False) + sparse_h_vals[start:end] = psf_vals_flat + + non_zero_mask = sparse_h_vals != 0 + return sparse_h_inds[:, non_zero_mask], sparse_h_vals[non_zero_mask] diff --git a/src/giant_python/bandsilo/hdf5.py b/src/giant_python/bandsilo/hdf5.py new file mode 100644 index 0000000..f6610b4 --- /dev/null +++ b/src/giant_python/bandsilo/hdf5.py @@ -0,0 +1,334 @@ +"""HDF5 struct (de)serialization for the BandSILo backend. + +BandRegistration.m (GIAnT-MATLAB) writes its outputs as HDF5 via +``saveStructToH5.m`` rather than the old ``.mat`` files. The readers below +mirror ``loadStructFromH5.m``: groups become nested dicts, string datasets +become ``str`` (or object arrays), and numeric datasets are returned in MATLAB +``size()`` orientation. MATLAB writes column-major (the ``/row_major`` flag is +``0`` or absent), so h5py reads datasets with axes reversed and we transpose to +recover the original layout. + +Ported from ``extractSLAP2IntegrationSources.py`` (``load_struct_from_h5``, +``load_alignment_data_h5``, ``_write_dict_to_h5group``, ``to_serializable``). +This module is intentionally private to :mod:`giant_python.bandsilo`; the +generic ``giant_python.io.hdf5`` helpers remain a separate concern. +""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import Union + +import h5py +import numpy as np + + +def to_serializable(val): + """Recursively convert numpy types/arrays to JSON-friendly Python values. + + Parameters + ---------- + val : object + A scalar, numpy value/array, or (possibly nested) dict/list/tuple. + + Returns + ------- + object + The same structure with numpy scalars converted to Python scalars and + numpy arrays converted to (nested) lists. + """ + if isinstance(val, dict): + return {k: to_serializable(v) for k, v in val.items()} + if isinstance(val, (list, tuple)): + return type(val)(to_serializable(v) for v in val) + if isinstance(val, np.ndarray): + return val.tolist() + if isinstance(val, np.generic): + return val.item() + return val + + +def _h5_is_string_dataset(ds: h5py.Dataset) -> bool: + """Return whether an h5py dataset holds (fixed/variable-length) strings.""" + if h5py.check_string_dtype(ds.dtype) is not None: + return True + return ds.dtype.kind in ("S", "U", "O") + + +def _decode_one(x) -> str: + """Decode a single bytes/str scalar to ``str``.""" + if isinstance(x, (bytes, np.bytes_)): + return x.decode("utf-8", "replace") + return str(x) + + +def _decode_h5_strings(val): + """Decode a scalar or ndarray of (variable-length) strings to ``str``. + + Parameters + ---------- + val : bytes, str, or ndarray + Raw value read from a string dataset. + + Returns + ------- + str or ndarray of object + Decoded scalar string, or an object array of decoded strings. + """ + if isinstance(val, (bytes, str, np.bytes_, np.str_)): + return _decode_one(val) + arr = np.asarray(val, dtype=object) + if arr.size == 0: + return arr + return np.vectorize(_decode_one, otypes=[object])(arr) + + +def _read_h5_dataset(ds: h5py.Dataset, row_major: bool): + """Read one dataset in MATLAB ``size()`` orientation. + + Scalar strings/paths (e.g. ``datadr``) are returned as plain ``str``; + string grids (e.g. ``filename``, ``fn_adata``) stay as object arrays with + MATLAB shape. + + Parameters + ---------- + ds : h5py.Dataset + The dataset to read. + row_major : bool + Whether the file was written row-major (``True``) or column-major. + + Returns + ------- + object + The decoded value (``str``, scalar, or ndarray). + """ + val = ds[()] + if _h5_is_string_dataset(ds): + val = _decode_h5_strings(val) + if isinstance(val, np.ndarray): + if (not row_major) and val.ndim >= 2: + val = val.T + if val.size == 1: + return val.reshape(-1)[0] + return val + val = np.asarray(val) + if (not row_major) and val.ndim >= 2: + val = val.T + return val + + +def _read_h5_group(grp: h5py.Group, row_major: bool) -> dict: + """Recursively read an h5py group into a nested dict.""" + out = {} + for key, item in grp.items(): + if key == "row_major": + continue + if isinstance(item, h5py.Group): + out[key] = _read_h5_group(item, row_major) + else: + out[key] = _read_h5_dataset(item, row_major) + return out + + +def load_struct_from_h5(path: Union[str, Path]) -> dict: + """Load a (possibly nested) MATLAB struct from HDF5 into a nested dict. + + Python port of GIAnT-MATLAB ``loadStructFromH5.m``. Honors the + ``/row_major`` layout flag (absent => column-major, so 2-D+ axes are + transposed to recover MATLAB/README orientation). + + Parameters + ---------- + path : str or Path + Path to the HDF5 file. + + Returns + ------- + dict + The nested struct as a dict of dicts / ndarrays / scalars. + """ + with h5py.File(path, "r") as f: + row_major = False + if "row_major" in f: + row_major = bool(np.asarray(f["row_major"][()]).reshape(-1)[0]) + return _read_h5_group(f, row_major) + + +def write_dict_to_h5group(grp: h5py.Group, d: dict) -> None: + """Write a (JSON-serializable) dict into an h5 group key-by-key. + + Nested dicts become subgroups; scalars/arrays become datasets; bools are + stored as uint8; empty/None values are skipped; anything unexpected is + stringified. + + Parameters + ---------- + grp : h5py.Group + Destination group. + d : dict + The dict to serialize. + """ + str_dt = h5py.string_dtype(encoding="utf-8") + for key, v in d.items(): + key = str(key) + if isinstance(v, dict): + write_dict_to_h5group(grp.create_group(key), v) + elif isinstance(v, bool): + grp.create_dataset(key, data=np.uint8(v)) + elif isinstance(v, str): + grp.create_dataset(key, data=v, dtype=str_dt) + elif isinstance(v, (int, float)): + grp.create_dataset(key, data=v) + elif isinstance(v, (list, tuple)): + arr = np.asarray(v) + if arr.size == 0: + continue + if arr.dtype.kind in ("U", "S", "O"): + grp.create_dataset( + key, + data=np.array([str(x) for x in v], dtype=object), + dtype=str_dt, + ) + else: + grp.create_dataset(key, data=arr) + elif v is None: + continue + else: + grp.create_dataset(key, data=str(v), dtype=str_dt) + + +def _reshape_1d(src: dict, key: str): + """Return ``src[key]`` flattened to 1-D, or ``None`` if missing.""" + v = src.get(key) + return None if v is None else np.asarray(v).reshape(-1) + + +def load_alignment_data_h5(path: Union[str, Path]) -> dict: + """Load a ``*_ALIGNMENTDATA.h5`` file (aData) into flat arrays/scalars. + + Mirrors the aData struct written by BandRegistration.m. Motion estimates + are stored as column vectors and online-motion shifts live under the nested + ``slap2`` group in the new format; everything is flattened to 1-D here. + + Parameters + ---------- + path : str or Path + Path to the ``*_ALIGNMENTDATA.h5`` file. + + Returns + ------- + dict + Keys ``DSframes``, ``motionDSr/c/z``, ``onlineYshift/Xshift/Zshift``, + ``numChannels``, ``alignHz``. + """ + s = load_struct_from_h5(path) + slap2 = s.get("slap2", {}) or {} + return { + "DSframes": _reshape_1d(s, "DSframes"), + "motionDSr": _reshape_1d(s, "motionDSr"), + "motionDSc": _reshape_1d(s, "motionDSc"), + "motionDSz": _reshape_1d(s, "motionDSz"), + "onlineYshift": _reshape_1d(slap2, "onlineMotionYshift"), + "onlineXshift": _reshape_1d(slap2, "onlineMotionXshift"), + "onlineZshift": _reshape_1d(slap2, "onlineMotionZshift"), + "numChannels": int(np.asarray(s["numChannels"]).reshape(-1)[0]), + "alignHz": float(np.asarray(s["alignHz"]).reshape(-1)[0]), + } + + +def _resolve_fn_adata(fn_adata: np.ndarray, moco_save_dr: str) -> np.ndarray: + """Resolve per-trial aData basenames to absolute paths under moco dir. + + Parameters + ---------- + fn_adata : ndarray of object, shape (n_dmds, n_trials) + Alignment-data basenames (may contain ``''``/``None``). + moco_save_dr : str + ``/motion_correction`` directory. + + Returns + ------- + ndarray of object, shape (n_dmds, n_trials) + Absolute paths (``''`` where the entry was empty). + """ + n_dmds, n_trials = fn_adata.shape + out = np.empty((n_dmds, n_trials), dtype=object) + for d in range(n_dmds): + for t in range(n_trials): + entry = fn_adata[d, t] + if entry in (None, ""): + out[d, t] = "" + else: + out[d, t] = os.path.join(moco_save_dr, str(entry)) + return out + + +def load_trial_table(path: Union[str, Path]) -> dict: + """Load and normalize a GIAnT ``trial_table.h5`` for BandSILo. + + Reads the trial table written by BandRegistration.m and assembles the + normalized structure consumed by the integration backend (mirrors the + setup block of ``main`` in ``extractSLAP2IntegrationSources.py``). + + Parameters + ---------- + path : str or Path + Path to the ``trial_table.h5`` file. + + Returns + ------- + dict + Keys: ``datadr``, ``result_dr``, ``moco_save_dr``, + ``annotation_save_dr``, ``src_extr_save_dr``, ``align_params``, + ``n_dmds``, ``n_trials``, and ``trial_table`` (a dict with + ``filename``, ``first_line``, ``last_line``, ``fn_adata`` (absolute + paths), ``datadr``, ``savedr``, ``ref_stack``). + """ + tt = load_struct_from_h5(path) + + datadr = str(tt["datadr"]) + result_dr = str(tt["savedr"]) + moco_save_dr = os.path.join(result_dr, "motion_correction") + + motion_correction = tt.get("motion_correction", {}) or {} + slap2_info = tt.get("slap2_info", {}) or {} + + align_params = { + key: to_serializable(val) + for key, val in ( + motion_correction.get("align_params", {}) or {} + ).items() + } + + filenames = np.atleast_2d(np.asarray(tt["filename"], dtype=object)) + first_line = np.atleast_2d(np.asarray(slap2_info["first_line"])) + last_line = np.atleast_2d(np.asarray(slap2_info["last_line"])) + fn_adata = np.atleast_2d( + np.asarray(motion_correction["fn_adata"], dtype=object) + ) + + n_dmds, n_trials = filenames.shape + fn_adata_full = _resolve_fn_adata(fn_adata, moco_save_dr) + + trial_table = { + "filename": filenames, + "first_line": first_line, + "last_line": last_line, + "fn_adata": fn_adata_full, + "datadr": datadr, + "savedr": result_dr, + "ref_stack": slap2_info.get("ref_stack", {}) or {}, + } + + return { + "datadr": datadr, + "result_dr": result_dr, + "moco_save_dr": moco_save_dr, + "annotation_save_dr": os.path.join(result_dr, "annotations"), + "src_extr_save_dr": os.path.join(result_dr, "source_extraction"), + "align_params": align_params, + "n_dmds": int(n_dmds), + "n_trials": int(n_trials), + "trial_table": trial_table, + } diff --git a/src/giant_python/models/params.py b/src/giant_python/models/params.py index 1c395d0..b6736a2 100644 --- a/src/giant_python/models/params.py +++ b/src/giant_python/models/params.py @@ -8,6 +8,7 @@ from __future__ import annotations +import math from dataclasses import dataclass from typing import Optional @@ -73,7 +74,42 @@ class SiloParams: draw_user_rois : bool Whether to prompt for user ROIs before extraction. analyze_hz : float or None - Temporal analysis rate, in Hz. + Temporal analysis rate, in Hz. When ``None``, the BandSILo backend + falls back to ``100``. + + Notes + ----- + The remaining attributes below are specific to the ``"integration"`` + (BandSILo) backend and mirror the GUI defaults in + ``extractSLAP2IntegrationSources.py``. They are ignored by the standard + (pixel-movie) SILo backend. + + decay_tau_s : float + Calcium-decay time constant, in seconds. + baseline_window_s : float + Rolling-baseline window for dF/F, in seconds. + denoise_window_s : float + Temporal denoising window, in seconds. + vif : float + Variance inflation factor for the noise model. + d_xy : int + Spatial downsampling / superpixel spacing, in pixels. + sparse_fac : float + NMF sparsity factor (the GUI exposes its natural-log; the default is + ``exp(-3)``). + peak_buffer : int + Peak exclusion buffer diameter, in pixels. + select_soma : bool + Whether to prompt for manual soma annotation before extraction. + max_workers : int + Number of worker processes for per-trial extraction. + num_channels : int or None + Number of acquisition channels; when ``None``, read from the aData + alignment file at runtime. + psf_dilation : int + Dilation size selecting the bundled ``dil-NN.tif`` PSF template. + operator : str + Operator name recorded in the output metadata. """ microscope: str = "slap2" @@ -87,6 +123,18 @@ class SiloParams: activity_channel: int = 0 draw_user_rois: bool = False analyze_hz: Optional[float] = None + decay_tau_s: float = 0.15 + baseline_window_s: float = 4.0 + denoise_window_s: float = 1.0 + vif: float = 1.38 + d_xy: int = 5 + sparse_fac: float = math.exp(-3.0) + peak_buffer: int = 3 + select_soma: bool = False + max_workers: int = 6 + num_channels: Optional[int] = None + psf_dilation: int = 17 + operator: str = "SLAP2 User" def set_params( diff --git a/tests/test_bandsilo_geometry.py b/tests/test_bandsilo_geometry.py new file mode 100644 index 0000000..cf5dc47 --- /dev/null +++ b/tests/test_bandsilo_geometry.py @@ -0,0 +1,176 @@ +"""Tests for giant_python.bandsilo.geometry.""" + +import os +import tempfile +import unittest + +import h5py +import numpy as np +import tifffile + +from giant_python.bandsilo.geometry import ( + _pad_to, + build_combined_psf, + build_sparse_h, + build_subsample_matrix_inds, + default_psf, + find_reference_file, + load_lookup_table, + load_psf, + load_reference_stack, + ref_pixs_to_drc, + threshold_and_crop_psf, +) + + +class TestRefPixsToDrc(unittest.TestCase): + """ref_pixs_to_drc maps flat pixels to (depth, column, row).""" + + def test_known_mapping(self): + """A hand-computed pixel decomposes to the expected d/c/r.""" + # C (per column) = 4, R (per row) = 5 -> plane = 20. + # pix = d*plane + c*C + r = 1*20 + 2*4 + 2 = 30 + d, c, r = ref_pixs_to_drc(np.array([30]), 4, 5) + self.assertEqual((int(d[0]), int(c[0]), int(r[0])), (1, 2, 2)) + + +class TestSubsampleMatrixInds(unittest.TestCase): + """build_subsample_matrix_inds picks median open pixels.""" + + def test_median_reference_pixel(self): + """Odd-length superpixels use the median; even-length use the middle.""" + all_ids = np.array([[1], [2]], dtype=np.int32) + mask = np.array( + [[10, 1], [12, 1], [14, 1], [20, 2], [22, 2]], dtype=np.int32 + ) + out = build_subsample_matrix_inds(all_ids, mask) + np.testing.assert_array_equal( + out, np.array([[11, 1], [21, 2]], dtype=np.int32) + ) + + def test_odd_length_uses_median_not_middle_index(self): + """Unsorted odd-length open pixels pick median, not middle by order.""" + all_ids = np.array([[1]], dtype=np.int32) + # 0-based open pixels: [13, 9, 11] -> median 11, middle index -> 9 + mask = np.array([[14, 1], [10, 1], [12, 1]], dtype=np.int32) + out = build_subsample_matrix_inds(all_ids, mask) + np.testing.assert_array_equal(out, np.array([[11, 1]], dtype=np.int32)) + + +class TestSparseH(unittest.TestCase): + """build_sparse_h assembles the PSF-convolution COO matrix.""" + + def test_known_values(self): + """Indices/values match a hand-computed single-superpixel case.""" + sub = np.array([[30, 1]], dtype=np.int32) # d=1, c=2, r=2 (C=4,R=5) + psf = np.array( + [[0.0, 1.0, 0.0], [2.0, 3.0, 4.0], [0.0, 5.0, 0.0]], + dtype=np.float32, + ) + inds, vals = build_sparse_h(sub, psf, 4, 5) + np.testing.assert_array_equal(inds[0], np.zeros(5, dtype=np.int32)) + np.testing.assert_array_equal( + inds[1], np.array([27, 31, 32, 33, 37], dtype=np.int32) + ) + np.testing.assert_array_equal( + vals, np.array([1, 2, 3, 4, 5], dtype=np.float32) + ) + + +class TestLookupTable(unittest.TestCase): + """load_lookup_table reshapes per-DMD lookup arrays.""" + + def test_load(self): + """Superpixel ids/mask/fastZ arrays load in expected shapes.""" + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "bandRegLookupTable.h5") + with h5py.File(path, "w") as f: + f.create_dataset("row_major", data=1) + p1 = f.create_group("Path1") + p1.create_dataset("allSuperPixelIDs", data=np.array([1, 2, 3])) + p1.create_dataset( + "sparseMaskInds", + data=np.array([[1, 1], [2, 1], [3, 2]]), + ) + p1.create_dataset("fastZ2RefZ", data=np.array([1, 2])) + out = load_lookup_table(path, 1) + self.assertEqual(out["allSuperPixelIDs"]["DMD1"].shape, (3, 1)) + self.assertEqual(out["sparseMaskInds"]["DMD1"].shape, (3, 2)) + self.assertEqual(out["fastZ2RefZ"]["DMD1"].shape, (2, 1)) + + +class TestReferenceFiles(unittest.TestCase): + """find_reference_file and load_reference_stack.""" + + def test_find_first_pattern_and_missing(self): + """CONFIG2 pattern is matched; an empty dir returns None.""" + with tempfile.TemporaryDirectory() as d: + name = "expt_DMD1_CONFIG2-REFERENCE.tif" + open(os.path.join(d, name), "wb").close() + self.assertTrue(find_reference_file(d, 0).endswith(name)) + with tempfile.TemporaryDirectory() as empty: + self.assertIsNone(find_reference_file(empty, 0)) + + def test_load_reference_stack(self): + """A synthetic REFERENCE tif reshapes to [channel, z, y, x].""" + meta = {"Path1": {"channels": np.array([1, 2])}} + with tempfile.TemporaryDirectory() as d: + data = (np.arange(6 * 4 * 4).reshape(6, 4, 4)).astype(np.float32) + tifffile.imwrite( + os.path.join(d, "expt_DMD1-REFERENCE.tif"), + data, + photometric="minisblack", + ) + stack, channels, ref_file = load_reference_stack(d, meta, 0) + self.assertEqual(stack.shape, (2, 3, 4, 4)) + self.assertEqual(list(channels), [1, 2]) + self.assertTrue(ref_file.endswith("expt_DMD1-REFERENCE.tif")) + + def test_load_reference_stack_missing(self): + """No REFERENCE file yields (None, channels, None).""" + meta = {"Path1": {"channels": np.array([1, 2])}} + with tempfile.TemporaryDirectory() as d: + stack, channels, ref_file = load_reference_stack(d, meta, 0) + self.assertIsNone(stack) + self.assertIsNone(ref_file) + self.assertEqual(list(channels), [1, 2]) + + +class TestPsf(unittest.TestCase): + """PSF loading, padding, thresholding, cropping.""" + + def test_default_psf(self): + """A bundled PSF template loads as a 2-D float32 array.""" + psf = default_psf(17) + self.assertEqual(psf.ndim, 2) + self.assertEqual(psf.dtype, np.float32) + + def test_pad_and_combine(self): + """build_combined_psf center-pads to common dims.""" + p1 = np.ones((3, 3), dtype=np.float32) + p2 = np.full((1, 1), 5.0, dtype=np.float32) + combined = build_combined_psf([p1, p2]) + self.assertEqual(combined.shape, (2, 3, 3)) + padded = _pad_to(p2, 3, 3) + self.assertEqual(padded.shape, (3, 3)) + + def test_threshold_and_crop(self): + """Sub-threshold pixels are zeroed and boundary zeros cropped.""" + arr = np.zeros((5, 5), dtype=np.float32) + arr[2, 2] = 10.0 + arr[1, 2] = arr[3, 2] = arr[2, 1] = arr[2, 3] = 1.0 + arr[0, 0] = 0.1 # below 10*exp(-3) ~ 0.498 -> zeroed + cropped = threshold_and_crop_psf(arr) + self.assertEqual(cropped.shape, (3, 3)) + self.assertEqual(cropped[1, 1], 10.0) + + def test_load_psf_from_assets(self): + """load_psf always builds per-DMD PSFs from bundled assets.""" + psf = load_psf(17, 2) + self.assertEqual(set(psf), {"DMD1", "DMD2"}) + self.assertEqual(psf["DMD1"].ndim, 2) + self.assertEqual(psf["DMD1"].dtype, np.float32) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_bandsilo_hdf5.py b/tests/test_bandsilo_hdf5.py new file mode 100644 index 0000000..74958d6 --- /dev/null +++ b/tests/test_bandsilo_hdf5.py @@ -0,0 +1,253 @@ +"""Tests for giant_python.bandsilo.hdf5.""" + +import os +import tempfile +import unittest + +import h5py +import numpy as np + +from giant_python.bandsilo.hdf5 import ( + _decode_h5_strings, + _decode_one, + _reshape_1d, + _resolve_fn_adata, + load_alignment_data_h5, + load_struct_from_h5, + load_trial_table, + to_serializable, + write_dict_to_h5group, +) + + +def _str_dt(): + """Return the h5py variable-length utf-8 string dtype.""" + return h5py.string_dtype(encoding="utf-8") + + +class TestToSerializable(unittest.TestCase): + """to_serializable converts numpy structures to plain Python.""" + + def test_dict_list_tuple(self): + """Nested dict/list/tuple are recursed into.""" + out = to_serializable({"a": [np.int64(1), (np.float64(2.0),)]}) + self.assertEqual(out, {"a": [1, (2.0,)]}) + self.assertIsInstance(out["a"][1], tuple) + + def test_ndarray_and_generic(self): + """ndarray -> list and numpy scalar -> Python scalar.""" + self.assertEqual(to_serializable(np.arange(3)), [0, 1, 2]) + self.assertEqual(to_serializable(np.float32(1.5)), 1.5) + + def test_passthrough(self): + """Plain Python values pass through unchanged.""" + self.assertEqual(to_serializable("x"), "x") + + +class TestDecodeHelpers(unittest.TestCase): + """_decode_one / _decode_h5_strings handle bytes, str, arrays.""" + + def test_decode_one(self): + """bytes decode to str; str stays str.""" + self.assertEqual(_decode_one(b"ab"), "ab") + self.assertEqual(_decode_one("cd"), "cd") + + def test_decode_scalar_and_array(self): + """Scalar bytes -> str; object array -> decoded object array.""" + self.assertEqual(_decode_h5_strings(b"z"), "z") + arr = _decode_h5_strings(np.array([b"a", b"b"], dtype=object)) + self.assertEqual(list(arr), ["a", "b"]) + + def test_decode_empty_array(self): + """An empty array is returned unchanged (no vectorize crash).""" + out = _decode_h5_strings(np.array([], dtype=object)) + self.assertEqual(out.size, 0) + + +class TestReadDataset(unittest.TestCase): + """load_struct_from_h5 read-side orientation and string handling.""" + + def test_scalar_string(self): + """A scalar string dataset reads back as plain str.""" + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "s.h5") + with h5py.File(path, "w") as f: + f.create_dataset("datadr", data="hello", dtype=_str_dt()) + out = load_struct_from_h5(path) + self.assertEqual(out["datadr"], "hello") + + def test_single_element_string_array(self): + """A 1-element string array collapses to a scalar str.""" + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "s.h5") + with h5py.File(path, "w") as f: + f.create_dataset( + "name", + data=np.array(["x"], dtype=object), + dtype=_str_dt(), + ) + out = load_struct_from_h5(path) + self.assertEqual(out["name"], "x") + + def test_string_grid_column_major_transposed(self): + """A 2-D string grid is transposed when column-major (no flag).""" + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "s.h5") + grid = np.array([["a", "b", "c"], ["d", "e", "f"]], dtype=object) + with h5py.File(path, "w") as f: + f.create_dataset("filename", data=grid, dtype=_str_dt()) + out = load_struct_from_h5(path) + self.assertEqual(out["filename"].shape, (3, 2)) + + def test_numeric_transpose_and_row_major(self): + """Numeric 2-D arrays honor the row_major flag.""" + arr = np.arange(6).reshape(2, 3) + with tempfile.TemporaryDirectory() as d: + col = os.path.join(d, "c.h5") + with h5py.File(col, "w") as f: + f.create_dataset("m", data=arr) + self.assertEqual(load_struct_from_h5(col)["m"].shape, (3, 2)) + + rowm = os.path.join(d, "r.h5") + with h5py.File(rowm, "w") as f: + f.create_dataset("row_major", data=1) + f.create_dataset("m", data=arr) + got = load_struct_from_h5(rowm)["m"] + self.assertEqual(got.shape, (2, 3)) + np.testing.assert_array_equal(got, arr) + + def test_nested_group(self): + """Nested groups become nested dicts.""" + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "s.h5") + with h5py.File(path, "w") as f: + f.create_dataset("row_major", data=1) + g = f.create_group("inner") + g.create_dataset("v", data=np.arange(3)) + out = load_struct_from_h5(path) + np.testing.assert_array_equal(out["inner"]["v"], np.arange(3)) + + +class TestWriteRoundTrip(unittest.TestCase): + """write_dict_to_h5group covers every value type.""" + + def test_round_trip(self): + """All supported value kinds survive a write/read round trip.""" + payload = { + "row_major": 1, + "flag": True, + "name": "expt", + "count": 7, + "scale": 2.5, + "nums": [1, 2, 3], + "words": ["a", "b"], + "empty": [], + "missing": None, + "weird": 1 + 2j, + "nested": {"x": 1}, + } + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "w.h5") + with h5py.File(path, "w") as f: + write_dict_to_h5group(f, payload) + out = load_struct_from_h5(path) + + self.assertNotIn("empty", out) + self.assertNotIn("missing", out) + self.assertNotIn("row_major", out) + self.assertEqual(out["name"], "expt") + self.assertEqual(int(out["count"]), 7) + self.assertAlmostEqual(float(out["scale"]), 2.5) + self.assertEqual(int(out["flag"]), 1) + np.testing.assert_array_equal(out["nums"], np.array([1, 2, 3])) + self.assertEqual(list(out["words"]), ["a", "b"]) + self.assertEqual(out["weird"], "(1+2j)") + self.assertEqual(int(out["nested"]["x"]), 1) + + +class TestReshapeAndAlignment(unittest.TestCase): + """_reshape_1d and load_alignment_data_h5.""" + + def test_reshape_1d(self): + """Missing key -> None; present key -> flattened array.""" + self.assertIsNone(_reshape_1d({}, "k")) + np.testing.assert_array_equal( + _reshape_1d({"k": np.array([[1], [2]])}, "k"), + np.array([1, 2]), + ) + + def test_load_alignment_data(self): + """aData loads flat arrays; missing online fields become None.""" + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "a.h5") + with h5py.File(path, "w") as f: + f.create_dataset("row_major", data=1) + f.create_dataset("DSframes", data=np.arange(4)) + f.create_dataset("motionDSr", data=np.zeros(4)) + f.create_dataset("motionDSc", data=np.zeros(4)) + f.create_dataset("motionDSz", data=np.zeros(4)) + f.create_dataset("numChannels", data=2) + f.create_dataset("alignHz", data=100.0) + out = load_alignment_data_h5(path) + self.assertEqual(out["numChannels"], 2) + self.assertAlmostEqual(out["alignHz"], 100.0) + self.assertEqual(out["DSframes"].shape, (4,)) + self.assertIsNone(out["onlineYshift"]) + + +class TestTrialTable(unittest.TestCase): + """_resolve_fn_adata and load_trial_table normalization.""" + + def test_resolve_fn_adata(self): + """Empty entries stay empty; others become absolute paths.""" + arr = np.array([["", "a.h5"]], dtype=object) + out = _resolve_fn_adata(arr, os.path.join("root", "moco")) + self.assertEqual(out[0, 0], "") + self.assertEqual(out[0, 1], os.path.join("root", "moco", "a.h5")) + + def _write_trial_table(self, path): + """Write a minimal synthetic trial_table.h5 (row-major).""" + fnames = np.array([["t1.dat", "t2.dat"]], dtype=object) + fn_adata = np.array([["t1_AD.h5", ""]], dtype=object) + with h5py.File(path, "w") as f: + f.create_dataset("row_major", data=1) + f.create_dataset("datadr", data="/data", dtype=_str_dt()) + f.create_dataset("savedr", data="/res", dtype=_str_dt()) + f.create_dataset("filename", data=fnames, dtype=_str_dt()) + si = f.create_group("slap2_info") + si.create_dataset("first_line", data=np.array([[1, 2]])) + si.create_dataset("last_line", data=np.array([[10, 20]])) + rs = si.create_group("ref_stack") + p1 = rs.create_group("Path1") + p1.create_dataset("channels", data=np.array([1, 2])) + mc = f.create_group("motion_correction") + mc.create_dataset("fn_adata", data=fn_adata, dtype=_str_dt()) + ap = mc.create_group("align_params") + ap.create_dataset("align_hz", data=80.0) + + def test_load_trial_table(self): + """Normalized dict has resolved paths, dims, and align params.""" + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "trial_table.h5") + self._write_trial_table(path) + out = load_trial_table(path) + + self.assertEqual(out["datadr"], "/data") + self.assertEqual(out["result_dr"], "/res") + self.assertEqual( + out["moco_save_dr"], os.path.join("/res", "motion_correction") + ) + self.assertEqual(out["n_dmds"], 1) + self.assertEqual(out["n_trials"], 2) + self.assertAlmostEqual(out["align_params"]["align_hz"], 80.0) + tt = out["trial_table"] + self.assertEqual(tt["fn_adata"][0, 1], "") + self.assertEqual( + tt["fn_adata"][0, 0], + os.path.join("/res", "motion_correction", "t1_AD.h5"), + ) + self.assertIn("Path1", tt["ref_stack"]) + + +if __name__ == "__main__": + unittest.main() From 9d001813f4b5a1cfe223493cd229d9526ecf0c53 Mon Sep 17 00:00:00 2001 From: Michael Xie Date: Thu, 16 Jul 2026 11:12:22 -0700 Subject: [PATCH 03/55] add code for params gui and user roi selection gui --- src/giant_python/bandsilo/gui.py | 694 ++++++++++++++++++++++++++++++ src/giant_python/models/params.py | 3 - tests/test_bandsilo_gui.py | 232 ++++++++++ 3 files changed, 926 insertions(+), 3 deletions(-) create mode 100644 src/giant_python/bandsilo/gui.py create mode 100644 tests/test_bandsilo_gui.py diff --git a/src/giant_python/bandsilo/gui.py b/src/giant_python/bandsilo/gui.py new file mode 100644 index 0000000..4ea38fa --- /dev/null +++ b/src/giant_python/bandsilo/gui.py @@ -0,0 +1,694 @@ +"""Parameter GUI, user-ROI annotation GUI, and annotation HDF5 IO. + +Ported from ``extractSLAP2IntegrationSources.py`` (parameter dialog ref +1013-1152, annotation helpers/IO ref 1155-1357, user-ROI selection GUI ref +1567-1741). The interactive dialogs (``tkinter`` parameter form, ``tkinter`` +label prompt, ``cv2`` ROI selector) import their GUI toolkits lazily so that +importing this module never requires a display; they are excluded from coverage +because they cannot run headless. The pure geometry/superpixel helpers and the +annotation reader/writer are fully testable and carry the numerical contract. +""" + +from __future__ import annotations + +import os +from dataclasses import replace +from pathlib import Path +from typing import Optional + +import h5py +import numpy as np + +from ..models import SiloParams + +# Mapping of parameter-GUI fields to SiloParams attributes. ``sparse_fac`` is +# handled separately (the GUI exposes its natural log). +_PARAM_GUI_FIELDS = ( + ("analyze_hz", "Analyze Hz:"), + ("decay_tau_s", "Decay Tau (s):"), + ("baseline_window_s", "Baseline Window (s):"), + ("denoise_window_s", "Denoise Window (s):"), + ("vif", "Variance Inflation Factor (VIF):"), + ("d_xy", "dXY:"), + ("peakth", "Peak Threshold:"), + ("peak_buffer", "Peak Buffer:"), + ("max_workers", "Max Workers:"), + ("operator", "Operator:"), +) + + +def _h5_scalar_str(ds: h5py.Dataset) -> str: + """Read an h5 scalar string dataset as a python ``str``.""" + v = ds[()] + if isinstance(v, np.ndarray): + v = v.reshape(-1)[0] + if isinstance(v, bytes): + return v.decode("utf-8", "replace") + return str(v) + + +def user_roi_superpixel_lists_from_masks( + roi_masks: list, + sp_fastz: np.ndarray, + sp_rows: np.ndarray, + sp_cols: np.ndarray, +) -> list: + """Return one superpixel-index list per user ROI, from boolean masks. + + ROIs may overlap: a superpixel inside several ROIs appears in each list. + + Parameters + ---------- + roi_masks : list of ndarray + Per-ROI boolean masks of shape ``(num_fast_z, Y, X)``. + sp_fastz, sp_rows, sp_cols : ndarray + Per-superpixel fast-Z / row / column indices into the mask. + + Returns + ------- + list of ndarray + For each ROI, the flat indices of superpixels falling inside it. + """ + return [ + np.flatnonzero(np.asarray(m)[sp_fastz, sp_rows, sp_cols]) + for m in roi_masks + ] + + +def compute_user_roi_geometry( + ref: np.ndarray, + fastz_to_refz: np.ndarray, + subsample_matrix_inds: np.ndarray, + motion_median: tuple, +) -> dict: + """Compute reference-image geometry for the user-ROI selection GUI. + + Derives the best display channel, the fast-Z -> reference-Z map (shifted by + the median Z motion), and the per-superpixel (fast-Z, row, column) indices + (shifted by the median row/column motion), plus a boolean superpixel mask. + + Parameters + ---------- + ref : ndarray + Reference stack for one DMD, shape ``[channels, z, Y, X]``. + fastz_to_refz : ndarray + Fast-Z -> reference-Z lookup (1-based), shape ``(num_fast_z, 1)``. + subsample_matrix_inds : ndarray + ``[ref_pixel (0-based), superpixel_id]`` rows for this DMD. + motion_median : tuple of int + Median ``(row, column, z)`` motion for this DMD. + + Returns + ------- + dict + Geometry fields consumed by the user-ROI selection GUI and the + annotation reader: ``ref``, ``num_ref_z``, ``yx_shape``, ``best_ch``, + ``z_map``, ``num_fast_z``, ``sp_fastz``, ``sp_rows``, ``sp_cols``, + ``sp_mask``. + """ + avg_motion_r, avg_motion_c, avg_motion_z = motion_median + + num_ref_z = ref.shape[1] + yx_shape = (ref.shape[2], ref.shape[3]) + plane = yx_shape[0] * yx_shape[1] + ch_means = [np.nanmean(ref[c]) for c in range(ref.shape[0])] + best_ch = int(np.argmax(ch_means)) if len(ch_means) > 0 else 0 + + z_map = np.array(fastz_to_refz + avg_motion_z).reshape(-1) - 1 + num_fast_z = z_map.shape[0] + + ref_pix = subsample_matrix_inds[:, 0] + sp_fastz = ref_pix // plane + sp_cols = avg_motion_c + (ref_pix - sp_fastz * plane) // yx_shape[0] + sp_rows = avg_motion_r + ref_pix % yx_shape[0] + + sp_mask = np.zeros((num_fast_z, *yx_shape), dtype=bool) + valid = ( + (sp_rows >= 0) + & (sp_rows < yx_shape[0]) + & (sp_cols >= 0) + & (sp_cols < yx_shape[1]) + & (sp_fastz >= 0) + & (sp_fastz < num_fast_z) + ) + if np.any(valid): + sp_mask[sp_fastz[valid], sp_rows[valid], sp_cols[valid]] = True + + return { + "ref": ref, + "num_ref_z": num_ref_z, + "yx_shape": yx_shape, + "best_ch": best_ch, + "z_map": z_map, + "num_fast_z": num_fast_z, + "sp_fastz": sp_fastz, + "sp_rows": sp_rows, + "sp_cols": sp_cols, + "sp_mask": sp_mask, + } + + +def save_annotations_h5( + dr: str, + roi_records_by_dmd: dict, + user_roi_masks: dict, + n_dmds: int, + ref_files: Optional[dict] = None, +) -> str: + """Write user ROI annotations to ``dr/annotations.h5``. + + Rectangles drawn in the OpenCV selector are stored as ``type='polygon'`` + with a 4-vertex ``position`` (``[y, x]``, 0-indexed) plus a per-ROI binary + ``mask`` (fast-Z, Y, X). ``roi_records_by_dmd[key]`` is a list (in ROI + order) of dicts with keys ``type``, ``label``, ``position``; the per-ROI + mask is taken from ``user_roi_masks[key][i]`` (ROIs may overlap). + + Parameters + ---------- + dr : str + Destination directory (``annotations.h5`` is written inside it). + roi_records_by_dmd : dict + ``{"DMD{N}": [record, ...]}`` ROI metadata. + user_roi_masks : dict + ``{"DMD{N}": [mask, ...]}`` per-ROI boolean masks. + n_dmds : int + Number of DMD paths. + ref_files : dict, optional + ``{"DMD{N}": reference_file_path}`` for provenance. + + Returns + ------- + str + The path to the written ``annotations.h5``. + """ + path = os.path.join(dr, "annotations.h5") + str_dt = h5py.string_dtype(encoding="utf-8") + ref_files = ref_files or {} + with h5py.File(path, "w") as f: + f["row_major"] = 1 + f["coords_zero_indexed"] = 1 + for dmd_ix in range(n_dmds): + dmd_key = f"DMD{dmd_ix + 1}" + grp = f.create_group(f"Path{dmd_ix + 1}") + + ref_file = ref_files.get(dmd_key) + grp.create_dataset( + "dr", + data=(str(Path(ref_file).parent) if ref_file else ""), + dtype=str_dt, + ) + grp.create_dataset( + "fn", + data=(Path(ref_file).name if ref_file else ""), + dtype=str_dt, + ) + + records = roi_records_by_dmd.get(dmd_key, []) + grp.create_dataset("n_rois", data=len(records)) + + mask_list = user_roi_masks.get(dmd_key, []) + for i, rec in enumerate(records): + _write_roi_group(grp, i, rec, mask_list, str_dt) + return path + + +def _write_roi_group( + grp: h5py.Group, + i: int, + rec: dict, + mask_list: list, + str_dt, +) -> None: + """Write a single ``roi_###`` subgroup (type/label/mask/position).""" + rgrp = grp.create_group(f"roi_{i:03d}") + rtype = rec.get("type", "polygon") + rgrp.create_dataset("type", data=rtype, dtype=str_dt) + rgrp.create_dataset( + "label", data=rec.get("label", f"ROI{i + 1}"), dtype=str_dt + ) + if i < len(mask_list): + roi_mask = np.asarray(mask_list[i]).astype(np.uint8) + rgrp.create_dataset( + "mask", data=roi_mask, compression="gzip", shuffle=True + ) + if rtype == "polygon" and "position" in rec: + rgrp.create_dataset( + "position", data=np.asarray(rec["position"], dtype=np.float64) + ) + + +def _empty_path_selection(dmd_key: str, out: dict) -> None: + """Record an empty (no-ROI) selection for one DMD across all outputs.""" + out["user_roi_masks"][dmd_key] = [] + out["user_roi_superpixels"][dmd_key] = [] + out["user_roi_labels"][dmd_key] = [] + out["roi_records"][dmd_key] = [] + + +def _read_roi_records(grp: h5py.Group, expected: tuple): + """Read all ``roi_###`` subgroups for one path. + + Parameters + ---------- + grp : h5py.Group + The ``Path{N}`` group. + expected : tuple + Expected mask shape ``(num_fast_z, Y, X)``. + + Returns + ------- + tuple + ``(roi_masks, labels, recs, bad_shape)`` where ``bad_shape`` is True if + any ROI mask shape did not match ``expected``. + """ + n_rois = 0 + if "n_rois" in grp: + n_rois = int(np.asarray(grp["n_rois"][()]).reshape(-1)[0]) + roi_masks, labels, recs = [], [], [] + for i in range(n_rois): + rgrp = grp.get(f"roi_{i:03d}") + if rgrp is None: + continue + lbl = _h5_scalar_str(rgrp["label"]) if "label" in rgrp else f"ROI{i+1}" + typ = _h5_scalar_str(rgrp["type"]) if "type" in rgrp else "polygon" + if "mask" in rgrp: + m = np.asarray(rgrp["mask"][()]) + if m.shape != expected: + return roi_masks, labels, recs, True + roi_masks.append(m > 0) + else: + roi_masks.append(np.zeros(expected, dtype=bool)) + labels.append(lbl) + rec = {"type": typ, "label": lbl} + if "position" in rgrp: + rec["position"] = np.asarray(rgrp["position"][()]) + recs.append(rec) + return roi_masks, labels, recs, False + + +def load_annotations_h5( + annotations_path: str, + n_dmds: int, + user_roi_geo: dict, +) -> tuple: + """Load user ROI annotations from ``annotations.h5`` when the file exists. + + Reconstructs, per path, the per-ROI superpixel lists, text labels, and raw + ROI records. Each ``Path`` group may be absent (treated as no selection). + If a per-ROI mask shape does not match the current geometry, that path is + treated as no selection. + + Parameters + ---------- + annotations_path : str + Path to ``annotations.h5``. + n_dmds : int + Number of DMD paths. + user_roi_geo : dict + Per-DMD geometry from :func:`compute_user_roi_geometry`. + + Returns + ------- + tuple + ``(skip_manual, user_roi_masks, user_roi_superpixels, + user_roi_labels, roi_records)``. ``skip_manual`` is True iff the file + was read and at least one path had >= 1 ROI. On a missing or unreadable + file, returns ``(False, {}, {}, {}, {})`` and the caller runs manual + selection. + """ + empty = (False, {}, {}, {}, {}) + if not os.path.exists(annotations_path): + return empty + + out = { + "user_roi_masks": {}, + "user_roi_superpixels": {}, + "user_roi_labels": {}, + "roi_records": {}, + } + any_valid = False + try: + with h5py.File(annotations_path, "r") as hf: + for dmd_ix in range(n_dmds): + if _load_one_path(hf, dmd_ix, user_roi_geo, out): + any_valid = True + except OSError as e: + print(f"Could not read {annotations_path}: {e}") + return empty + + if not any_valid: + return empty + return ( + True, + out["user_roi_masks"], + out["user_roi_superpixels"], + out["user_roi_labels"], + out["roi_records"], + ) + + +def _load_one_path( + hf: h5py.File, + dmd_ix: int, + user_roi_geo: dict, + out: dict, +) -> bool: + """Load one path's annotations into ``out``; return True if it had ROIs.""" + dmd_key = f"DMD{dmd_ix + 1}" + path_key = f"Path{dmd_ix + 1}" + geo = user_roi_geo[dmd_key] + expected = (geo["num_fast_z"], geo["yx_shape"][0], geo["yx_shape"][1]) + + grp = hf.get(path_key) + if grp is None: + _empty_path_selection(dmd_key, out) + return False + + roi_masks, labels, recs, bad_shape = _read_roi_records(grp, expected) + if bad_shape: + print( + f"annotations.h5: {path_key} ROI mask shape mismatch " + f"(expected {expected}); treating this path as no selection." + ) + _empty_path_selection(dmd_key, out) + return False + + out["user_roi_masks"][dmd_key] = roi_masks + out["user_roi_superpixels"][dmd_key] = ( + user_roi_superpixel_lists_from_masks( + roi_masks, geo["sp_fastz"], geo["sp_rows"], geo["sp_cols"] + ) + ) + out["user_roi_labels"][dmd_key] = labels + out["roi_records"][dmd_key] = recs + if len(recs) > 0: + print(f"Loaded {len(recs)} ROI(s) for {path_key} from annotations.h5") + return True + return False + + +def run_parameter_gui( + params: Optional[SiloParams] = None, +) -> Optional[SiloParams]: # pragma: no cover - interactive tkinter dialog + """Open the tkinter parameter form and return an updated ``SiloParams``. + + The form is pre-filled from ``params`` (or :class:`SiloParams` defaults). + ``sparse_fac`` is edited on a natural-log scale. Returns ``None`` if the + user cancels. + + Parameters + ---------- + params : SiloParams, optional + Parameters to pre-fill; defaults to a fresh :class:`SiloParams`. + + Returns + ------- + SiloParams or None + The updated parameters, or ``None`` if cancelled. + """ + import tkinter as tk + from tkinter import messagebox, ttk + + base = params or SiloParams() + root = tk.Tk() + root.title("SLAP2 Analysis Parameters") + frame = ttk.Frame(root, padding="10") + frame.grid(row=0, column=0) + + entries = {} + row = 0 + for attr, label in _PARAM_GUI_FIELDS: + ttk.Label(frame, text=label).grid(row=row, column=0, sticky=tk.W) + var = tk.StringVar(value=str(getattr(base, attr))) + ttk.Entry(frame, textvariable=var, width=15).grid(row=row, column=1) + entries[attr] = var + row += 1 + + ttk.Label(frame, text="Sparse Factor (log):").grid( + row=row, column=0, sticky=tk.W + ) + sparse_var = tk.StringVar(value=str(float(np.log(base.sparse_fac)))) + ttk.Entry(frame, textvariable=sparse_var, width=15).grid(row=row, column=1) + row += 1 + + ttk.Label(frame, text="Draw User ROIs?").grid( + row=row, column=0, sticky=tk.W + ) + draw_rois_var = tk.BooleanVar(value=base.draw_user_rois) + ttk.Checkbutton(frame, variable=draw_rois_var).grid(row=row, column=1) + row += 1 + + result = {"params": None} + + def on_ok(): + """Validate entries and store an updated SiloParams, then close.""" + try: + updates = { + "analyze_hz": float(entries["analyze_hz"].get()), + "decay_tau_s": float(entries["decay_tau_s"].get()), + "baseline_window_s": float(entries["baseline_window_s"].get()), + "denoise_window_s": float(entries["denoise_window_s"].get()), + "vif": float(entries["vif"].get()), + "d_xy": int(entries["d_xy"].get()), + "peakth": float(entries["peakth"].get()), + "peak_buffer": int(entries["peak_buffer"].get()), + "max_workers": int(entries["max_workers"].get()), + "operator": entries["operator"].get(), + "sparse_fac": float(np.exp(float(sparse_var.get()))), + "draw_user_rois": bool(draw_rois_var.get()), + } + result["params"] = replace(base, **updates) + root.destroy() + except ValueError as e: + messagebox.showerror("Error", f"Invalid parameter values: {e}") + + def on_cancel(): + """Discard edits and close the dialog.""" + root.destroy() + + button_frame = ttk.Frame(frame) + button_frame.grid(row=row, column=0, columnspan=2, pady=10) + ttk.Button(button_frame, text="OK", command=on_ok).pack(side=tk.LEFT) + ttk.Button(button_frame, text="Cancel", command=on_cancel).pack( + side=tk.LEFT + ) + + root.grab_set() + root.mainloop() + return result["params"] + + +def _ask_roi_label( + roi_num: int, path_num: int, plane_num: int +) -> str: # pragma: no cover - interactive tkinter dialog + """Modal tkinter prompt for one ROI's label; defaults to ``ROI{n}``.""" + import tkinter as tk + from tkinter import simpledialog + + text = None + try: + root = tk.Tk() + root.withdraw() + root.attributes("-topmost", True) + text = simpledialog.askstring( + "ROI label", + f"Label for ROI {roi_num} (Path {path_num}, " + f"fast-Z plane {plane_num}):", + parent=root, + ) + root.destroy() + except Exception as e: # noqa: BLE001 - GUI best-effort + print(f"Could not prompt for ROI label ({e}); using default.") + if not text or not text.strip(): + return f"ROI{roi_num}" + return text.strip() + + +def run_user_roi_selection( + user_roi_geo: dict, + n_dmds: int, +) -> dict: # pragma: no cover - interactive cv2 GUI + """Interactively select user ROIs on each DMD's reference image. + + Scrolls through fast-Z planes and lets the user draw rectangles (stored as + polygons). Returns per-DMD ``user_roi_masks``, ``user_roi_superpixels``, + ``user_roi_labels``, and ``roi_records`` for writing via + :func:`save_annotations_h5`. + + Parameters + ---------- + user_roi_geo : dict + Per-DMD geometry from :func:`compute_user_roi_geometry`. + n_dmds : int + Number of DMD paths. + + Returns + ------- + dict + ``{"user_roi_masks", "user_roi_superpixels", "user_roi_labels", + "roi_records"}``. + """ + import cv2 + + out = { + "user_roi_masks": {}, + "user_roi_superpixels": {}, + "user_roi_labels": {}, + "roi_records": {}, + } + for dmd_ix in range(n_dmds): + dmd_key = f"DMD{dmd_ix + 1}" + g = user_roi_geo[dmd_key] + roi_masks, roi_records = _select_rois_for_dmd(cv2, dmd_ix, g) + out["user_roi_masks"][dmd_key] = roi_masks + out["user_roi_superpixels"][dmd_key] = ( + user_roi_superpixel_lists_from_masks( + roi_masks, g["sp_fastz"], g["sp_rows"], g["sp_cols"] + ) + ) + out["user_roi_labels"][dmd_key] = [r["label"] for r in roi_records] + out["roi_records"][dmd_key] = roi_records + return out + + +def _normalize_plane( + plane: np.ndarray, +) -> np.ndarray: # pragma: no cover - display helper + """Percentile-normalize a reference plane to an 8-bit image.""" + im = np.nan_to_num(plane, nan=0.0) + vmin = np.percentile(im, 1) + vmax = np.percentile(im, 99.5) + if not np.isfinite(vmin): + vmin = float(np.nanmin(im)) if np.any(np.isfinite(im)) else 0.0 + if not np.isfinite(vmax): + vmax = float(np.nanmax(im)) if np.any(np.isfinite(im)) else 1.0 + if vmax <= vmin: + vmax = vmin + 1.0 + im8 = np.clip((im - vmin) / (vmax - vmin), 0, 1) + return (im8 * 255).astype(np.uint8) + + +def _select_rois_for_dmd( + cv2, dmd_ix: int, g: dict +): # pragma: no cover - interactive cv2 GUI + """Run the interactive ROI selector for one DMD; return masks + records.""" + ref = g["ref"] + num_ref_z = g["num_ref_z"] + yx_shape = g["yx_shape"] + best_ch = g["best_ch"] + z_map = g["z_map"] + num_fast_z = g["num_fast_z"] + sp_mask = g["sp_mask"] + + roi_masks = [] + roi_records = [] + roi_union = np.zeros((num_fast_z, *yx_shape), dtype=bool) + + window_name = f"Select user ROI(s) DMD{dmd_ix + 1}" + cv2.namedWindow(window_name, cv2.WINDOW_NORMAL) + cv2.resizeWindow(window_name, 800, 500) + has_z_trackbar = num_fast_z > 1 + if has_z_trackbar: + cv2.createTrackbar("z", window_name, 0, num_fast_z - 1, lambda v: None) + + curr_fz = 0 + while True: + if has_z_trackbar: + curr_fz = int( + np.clip( + cv2.getTrackbarPos("z", window_name), 0, num_fast_z - 1 + ) + ) + refz = int(np.clip(z_map[curr_fz], 0, max(0, num_ref_z - 1))) + im8 = _normalize_plane(ref[best_ch, refz]) + disp = _compose_display(cv2, im8, sp_mask[curr_fz], roi_union[curr_fz]) + cv2.imshow(window_name, disp) + + keycode = cv2.waitKey(50) & 0xFF + if keycode == ord("e"): + _edit_rois( + cv2, + im8, + sp_mask[curr_fz], + curr_fz, + dmd_ix, + num_fast_z, + yx_shape, + roi_masks, + roi_union, + roi_records, + ) + elif keycode == ord("n"): + curr_fz = min(curr_fz + 1, num_fast_z - 1) + if has_z_trackbar: + cv2.setTrackbarPos("z", window_name, curr_fz) + elif keycode == ord("p"): + curr_fz = max(curr_fz - 1, 0) + if has_z_trackbar: + cv2.setTrackbarPos("z", window_name, curr_fz) + elif keycode in (27, ord("q")): + break + + cv2.destroyWindow(window_name) + return roi_masks, roi_records + + +def _compose_display( + cv2, im8: np.ndarray, sp_plane: np.ndarray, roi_plane: np.ndarray +): # pragma: no cover - display helper + """Overlay superpixel (green) and ROI-union (red) masks on the plane.""" + disp = cv2.cvtColor(im8, cv2.COLOR_GRAY2BGR) + if np.any(sp_plane): + sp_color = np.zeros_like(disp) + sp_color[sp_plane] = (0, 255, 0) + disp = cv2.addWeighted(disp, 0.75, sp_color, 0.25, 0) + if roi_plane.any(): + roi_color = np.zeros_like(disp) + roi_color[roi_plane] = (0, 0, 255) + disp = cv2.addWeighted(disp, 0.7, roi_color, 0.3, 0) + return disp + + +def _edit_rois( + cv2, + im8, + sp_plane, + curr_fz, + dmd_ix, + num_fast_z, + yx_shape, + roi_masks, + roi_union, + roi_records, +): # pragma: no cover - interactive cv2 GUI + """Prompt the user to draw ROI rectangles and append them in place.""" + edit_name = f"Edit ROIs z={curr_fz + 1}" + edit_disp = _compose_display( + cv2, im8, sp_plane, np.zeros(yx_shape, dtype=bool) + ) + rois = cv2.selectROIs( + edit_name, edit_disp, showCrosshair=True, fromCenter=False + ) + cv2.resizeWindow(edit_name, 800, 500) + if rois is not None and len(rois) > 0: + for x, y, w, h in rois: + roi_num = len(roi_masks) + 1 + y_stop, x_stop = y + h, x + w + m = np.zeros((num_fast_z, *yx_shape), dtype=bool) + m[curr_fz, y:y_stop, x:x_stop] = True + roi_masks.append(m) + roi_union[curr_fz, y:y_stop, x:x_stop] = True + label_text = _ask_roi_label(roi_num, dmd_ix + 1, curr_fz + 1) + position = np.array( + [[y, x], [y, x + w], [y + h, x + w], [y + h, x]], + dtype=np.float64, + ) + roi_records.append( + { + "type": "polygon", + "label": label_text, + "position": position, + } + ) + print(f"Added {len(rois)} ROI(s) at fast-Z {curr_fz + 1}") + cv2.destroyWindow(edit_name) diff --git a/src/giant_python/models/params.py b/src/giant_python/models/params.py index b6736a2..f0a691d 100644 --- a/src/giant_python/models/params.py +++ b/src/giant_python/models/params.py @@ -99,8 +99,6 @@ class SiloParams: ``exp(-3)``). peak_buffer : int Peak exclusion buffer diameter, in pixels. - select_soma : bool - Whether to prompt for manual soma annotation before extraction. max_workers : int Number of worker processes for per-trial extraction. num_channels : int or None @@ -130,7 +128,6 @@ class SiloParams: d_xy: int = 5 sparse_fac: float = math.exp(-3.0) peak_buffer: int = 3 - select_soma: bool = False max_workers: int = 6 num_channels: Optional[int] = None psf_dilation: int = 17 diff --git a/tests/test_bandsilo_gui.py b/tests/test_bandsilo_gui.py new file mode 100644 index 0000000..b2f5fd4 --- /dev/null +++ b/tests/test_bandsilo_gui.py @@ -0,0 +1,232 @@ +"""Tests for giant_python.bandsilo.gui (pure + annotation-IO paths).""" + +import os +import tempfile +import unittest + +import h5py +import numpy as np + +from giant_python.bandsilo.gui import ( + _h5_scalar_str, + _read_roi_records, + compute_user_roi_geometry, + load_annotations_h5, + save_annotations_h5, + user_roi_superpixel_lists_from_masks, +) + + +def _str_dt(): + """Return the h5py variable-length utf-8 string dtype.""" + return h5py.string_dtype(encoding="utf-8") + + +class TestSuperpixelLists(unittest.TestCase): + """user_roi_superpixel_lists_from_masks selects per-ROI superpixels.""" + + def test_overlapping_rois(self): + """A superpixel inside two ROIs appears in both lists.""" + sp_fastz = np.array([0, 0, 1]) + sp_rows = np.array([1, 2, 0]) + sp_cols = np.array([1, 2, 0]) + mask_a = np.zeros((2, 4, 5), dtype=bool) + mask_a[0, 1, 1] = True + mask_a[0, 2, 2] = True + mask_b = np.zeros((2, 4, 5), dtype=bool) + mask_b[0, 2, 2] = True + out = user_roi_superpixel_lists_from_masks( + [mask_a, mask_b], sp_fastz, sp_rows, sp_cols + ) + np.testing.assert_array_equal(out[0], np.array([0, 1])) + np.testing.assert_array_equal(out[1], np.array([1])) + + +class TestComputeUserRoiGeometry(unittest.TestCase): + """compute_user_roi_geometry derives channel/z-map/superpixel indices.""" + + def _ref(self): + """Return a 2-channel ref stack with channel 1 brighter.""" + ref = np.zeros((2, 3, 4, 5), dtype=np.float32) + ref[1] = 10.0 + return ref + + def test_no_motion(self): + """Zero motion: z_map is fastZ-1; indices from pixel decomposition.""" + ref = self._ref() + fastz_to_refz = np.array([[1], [2], [3]]) + sub = np.array([[22, 1]], dtype=np.int32) + geo = compute_user_roi_geometry(ref, fastz_to_refz, sub, (0, 0, 0)) + self.assertEqual(geo["best_ch"], 1) + self.assertEqual(geo["num_ref_z"], 3) + self.assertEqual(geo["yx_shape"], (4, 5)) + self.assertEqual(geo["num_fast_z"], 3) + np.testing.assert_array_equal(geo["z_map"], np.array([0, 1, 2])) + np.testing.assert_array_equal(geo["sp_fastz"], np.array([1])) + np.testing.assert_array_equal(geo["sp_cols"], np.array([0])) + np.testing.assert_array_equal(geo["sp_rows"], np.array([2])) + self.assertTrue(geo["sp_mask"][1, 2, 0]) + self.assertEqual(geo["sp_mask"].sum(), 1) + + def test_with_motion(self): + """Nonzero motion shifts z_map and row/column indices.""" + ref = self._ref() + fastz_to_refz = np.array([[1], [2], [3]]) + sub = np.array([[22, 1]], dtype=np.int32) + geo = compute_user_roi_geometry(ref, fastz_to_refz, sub, (1, 1, 1)) + np.testing.assert_array_equal(geo["z_map"], np.array([1, 2, 3])) + np.testing.assert_array_equal(geo["sp_rows"], np.array([3])) + np.testing.assert_array_equal(geo["sp_cols"], np.array([1])) + + def test_all_invalid_leaves_mask_empty(self): + """Out-of-bounds superpixels leave sp_mask all False.""" + ref = self._ref() + fastz_to_refz = np.array([[1]]) + sub = np.array([[100, 1]], dtype=np.int32) + geo = compute_user_roi_geometry(ref, fastz_to_refz, sub, (0, 0, 0)) + self.assertEqual(geo["sp_mask"].sum(), 0) + + def test_empty_channels(self): + """A zero-channel ref falls back to best_ch = 0.""" + ref = np.zeros((0, 3, 4, 5), dtype=np.float32) + geo = compute_user_roi_geometry( + ref, np.array([[1]]), np.zeros((0, 2), dtype=np.int32), (0, 0, 0) + ) + self.assertEqual(geo["best_ch"], 0) + + +class TestH5ScalarStr(unittest.TestCase): + """_h5_scalar_str decodes scalar/array/bytes string datasets.""" + + def test_variants(self): + """Bytes and 1-element arrays decode to plain str.""" + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "s.h5") + with h5py.File(path, "w") as f: + f.create_dataset("a", data="hi", dtype=_str_dt()) + f.create_dataset( + "b", data=np.array(["x"], dtype=object), dtype=_str_dt() + ) + f.create_dataset("c", data=5) + with h5py.File(path, "r") as f: + self.assertEqual(_h5_scalar_str(f["a"]), "hi") + self.assertEqual(_h5_scalar_str(f["b"]), "x") + self.assertEqual(_h5_scalar_str(f["c"]), "5") + + +class TestAnnotationsRoundTrip(unittest.TestCase): + """save_annotations_h5 / load_annotations_h5 round trip.""" + + def _geo(self): + """Return a minimal single-DMD geometry dict.""" + return { + "DMD1": { + "num_fast_z": 2, + "yx_shape": (4, 5), + "sp_fastz": np.array([0]), + "sp_rows": np.array([1]), + "sp_cols": np.array([1]), + } + } + + def _records_and_masks(self): + """Return one ROI record + matching mask for DMD1.""" + mask = np.zeros((2, 4, 5), dtype=bool) + mask[0, 1, 1] = True + records = { + "DMD1": [ + { + "type": "polygon", + "label": "user_roi_1", + "position": np.array( + [[1, 1], [1, 3], [3, 3], [3, 1]], dtype=np.float64 + ), + } + ] + } + return records, {"DMD1": [mask]} + + def test_round_trip(self): + """A saved ROI reloads with matching label, mask, and superpixels.""" + records, masks = self._records_and_masks() + with tempfile.TemporaryDirectory() as d: + path = save_annotations_h5( + d, records, masks, 1, ref_files={"DMD1": "/data/ref.tif"} + ) + self.assertTrue(os.path.exists(path)) + skip, m, sps, labels, recs = load_annotations_h5( + path, 1, self._geo() + ) + self.assertTrue(skip) + self.assertEqual(labels["DMD1"], ["user_roi_1"]) + self.assertTrue(m["DMD1"][0][0, 1, 1]) + np.testing.assert_array_equal(sps["DMD1"][0], np.array([0])) + self.assertEqual(recs["DMD1"][0]["type"], "polygon") + self.assertIn("position", recs["DMD1"][0]) + + def test_missing_file(self): + """A missing annotations file returns the empty result.""" + out = load_annotations_h5("/nope/annotations.h5", 1, self._geo()) + self.assertEqual(out, (False, {}, {}, {}, {})) + + def test_missing_path_group(self): + """A file with no Path group yields no valid selection.""" + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "annotations.h5") + with h5py.File(path, "w") as f: + f["row_major"] = 1 + out = load_annotations_h5(path, 1, self._geo()) + self.assertEqual(out, (False, {}, {}, {}, {})) + + def test_bad_mask_shape(self): + """A mask shape mismatch treats the path as no selection.""" + records, masks = self._records_and_masks() + with tempfile.TemporaryDirectory() as d: + path = save_annotations_h5(d, records, masks, 1) + geo = self._geo() + geo["DMD1"]["num_fast_z"] = 3 + out = load_annotations_h5(path, 1, geo) + self.assertEqual(out, (False, {}, {}, {}, {})) + + def test_empty_records_not_valid(self): + """A path with zero ROIs is read but not marked valid.""" + with tempfile.TemporaryDirectory() as d: + path = save_annotations_h5(d, {"DMD1": []}, {"DMD1": []}, 1) + out = load_annotations_h5(path, 1, self._geo()) + self.assertEqual(out, (False, {}, {}, {}, {})) + + def test_unreadable_file(self): + """A non-HDF5 file that exists returns the empty result.""" + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "annotations.h5") + with open(path, "w") as fh: + fh.write("not an hdf5 file") + out = load_annotations_h5(path, 1, self._geo()) + self.assertEqual(out, (False, {}, {}, {}, {})) + + +class TestReadRoiRecords(unittest.TestCase): + """_read_roi_records handles missing masks, labels, and gaps.""" + + def test_defaults_and_missing_mask(self): + """Missing label/type default; missing mask -> zero mask.""" + expected = (2, 4, 5) + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "a.h5") + with h5py.File(path, "w") as f: + grp = f.create_group("Path1") + grp.create_dataset("n_rois", data=2) + grp.create_group("roi_000") + with h5py.File(path, "r") as f: + masks, labels, recs, bad = _read_roi_records( + f["Path1"], expected + ) + self.assertFalse(bad) + self.assertEqual(labels, ["ROI1"]) + self.assertEqual(recs[0]["type"], "polygon") + self.assertEqual(masks[0].shape, expected) + self.assertFalse(masks[0].any()) + + +if __name__ == "__main__": + unittest.main() From a81767cf3bbb29e5585704955a65fedd610609d9 Mon Sep 17 00:00:00 2001 From: Michael Xie Date: Thu, 16 Jul 2026 11:54:38 -0700 Subject: [PATCH 04/55] implement functions for loading lowres slap2 data --- src/giant_python/bandsilo/trial_data.py | 450 ++++++++++++++++++++++++ tests/test_bandsilo_trial_data.py | 280 +++++++++++++++ 2 files changed, 730 insertions(+) create mode 100644 src/giant_python/bandsilo/trial_data.py create mode 100644 tests/test_bandsilo_trial_data.py diff --git a/src/giant_python/bandsilo/trial_data.py b/src/giant_python/bandsilo/trial_data.py new file mode 100644 index 0000000..8910cb1 --- /dev/null +++ b/src/giant_python/bandsilo/trial_data.py @@ -0,0 +1,450 @@ +"""Read low-resolution superpixel activity for integration (band-scan) trials. + +Ported from ``extractSLAP2IntegrationSources.py`` (``get_trial_data`` ref +715-842, ``nearest_interp``/``fast_dilation`` ref 166-205, and the per-trial +low-res caching block ref 1851-1892). The SLAP2 file open (``slap2_utils``) +and the OpenCV dilation fallback import their heavy dependencies lazily so +importing this module stays cheap. The accumulation core, downsample-frame +grid, interpolation, fast dilation, and the low-res assembly/cache IO are all +testable without ``slap2_utils``. +""" + +from __future__ import annotations + +import os +import re +from typing import Optional + +import numpy as np + +from .hdf5 import load_alignment_data_h5 + + +def nearest_interp( + x: np.ndarray, xp: np.ndarray, yp: np.ndarray +) -> np.ndarray: + """Nearest-neighbor interpolation (MATLAB ``interp1(...,'nearest')``). + + Parameters + ---------- + x : ndarray + Query points. + xp : ndarray + Sample positions (assumed sorted ascending). + yp : ndarray + Sample values aligned with ``xp``. + + Returns + ------- + ndarray + ``yp`` sampled at the nearest ``xp`` for each ``x``. + """ + if len(xp) == 1: + return yp + x_bds = xp[:-1] / 2.0 + xp[1:] / 2.0 + idx = np.searchsorted(x_bds, x, side="left") + idx = np.clip(idx, 0, len(xp) - 1) + return yp[idx] + + +def fast_dilation( + mask: np.ndarray, + kernel: Optional[np.ndarray] = None, + iterations: int = 1, +) -> np.ndarray: + """Binary-dilate the trailing two axes of ``mask``. + + The common 3x3 / single-iteration case is handled with a pure-numpy shift + accumulation (fast, no OpenCV); other cases fall back to per-slice + ``cv2.dilate`` over all leading axes. + + Parameters + ---------- + mask : ndarray + Boolean (or 0/1) array; the last two axes are the image plane. + kernel : ndarray, optional + Structuring element; defaults to a 7x7 block. + iterations : int + Number of dilation iterations (generic path only). + + Returns + ------- + ndarray of bool + The dilated mask. + """ + if kernel is None: + kernel = np.ones((7, 7), np.uint8) + + if iterations == 1 and kernel.shape == (3, 3) and np.all(kernel): + m = mask.astype(bool, copy=False) + pad = [(0, 0)] * (m.ndim - 2) + [(1, 1), (1, 1)] + p = np.pad(m, pad, mode="constant", constant_values=False) + out = np.zeros_like(m, dtype=bool) + h, w = m.shape[-2], m.shape[-1] + for dr in range(3): + r_end = dr + h + for dc in range(3): + c_end = dc + w + out |= p[..., dr:r_end, dc:c_end] + return out + + import cv2 + + out = np.empty_like(mask, dtype=bool) + for idx in np.ndindex(mask.shape[:-2]): + out[idx] = cv2.dilate( + mask[idx].astype(np.uint8, copy=False), + kernel, + iterations=iterations, + ).astype(bool, copy=False) + return out + + +def compute_ds_frames( + first_line: float, last_line: float, dt: float +) -> np.ndarray: + """Return the downsampled-frame line grid for one trial. + + Parameters + ---------- + first_line, last_line : float + First/last acquisition line (1-based, inclusive). + dt : float + Downsample step, in lines (``1 / samp_freq / line_period_s``). + + Returns + ------- + ndarray + ``ceil`` of the line positions of each downsampled frame. + """ + return np.ceil(np.arange(first_line, last_line + 1, dt)) + + +def _build_time_windows( + ds_frames: np.ndarray, dt: float, lines_per_cycle: int, num_cycles: int +): + """Return per-frame time windows and their line/cycle index arrays.""" + dt_read = max(3 * dt, lines_per_cycle) + max_line = num_cycles * lines_per_cycle + time_windows = [ + np.arange( + max(1, np.floor(f - dt_read)), + min(np.ceil(f + dt_read), max_line) + 1, + ) + for f in ds_frames + ] + line_indices = [(tw - 1) % lines_per_cycle + 1 for tw in time_windows] + cycle_indices = [ + np.floor((tw - 1) / lines_per_cycle) + 1 for tw in time_windows + ] + return time_windows, line_indices, cycle_indices + + +def _build_line_cycle_cache(line_indices: list, cycle_indices: list): + """Return unique (line, cycle) arrays and their ``(line, cycle)`` map.""" + pairs = set() + for lines, cycles in zip(line_indices, cycle_indices): + for li, ci in zip(lines, cycles): + pairs.add((int(li), int(ci))) + all_lines = np.array([p[0] for p in pairs]) + all_cycles = np.array([p[1] for p in pairs]) + mapping = { + (int(li), int(ci)): i + for i, (li, ci) in enumerate(zip(all_lines, all_cycles)) + } + return all_lines, all_cycles, mapping + + +def _accumulate_line( + acc: dict, + line_data: np.ndarray, + positions: np.ndarray, + z_idx: np.ndarray, + super_pixel_ids: np.ndarray, + weight: float, + ds_ix: int, +) -> None: + """Accumulate one line's weighted superpixel data into ``acc`` in place.""" + lookup_values = positions * 100 + z_idx + matching_mask = np.isin(super_pixel_ids, lookup_values) + matching_indices = np.where(matching_mask)[0] + if len(matching_indices) == 0: + return + value_to_pos = dict( + zip(lookup_values.astype(np.uint32), range(len(lookup_values))) + ) + matched = [ + value_to_pos[int(super_pixel_ids[idx])] for idx in matching_indices + ] + acc["data"][matching_indices, ds_ix] += line_data[matched, 0] * weight + acc["data_count"][matching_indices, ds_ix] += weight + if acc["data2"] is not None: + acc["data2"][matching_indices, ds_ix] += line_data[matched, 1] * weight + acc["data_count2"][matching_indices, ds_ix] += weight + + +def accumulate_superpixel_data( + data_file, + ds_frames: np.ndarray, + dt: float, + super_pixel_ids: np.ndarray, + num_channels: int, + all_channels: bool, +) -> dict: + """Accumulate weighted per-superpixel activity over the DS-frame grid. + + Port of the core loop of ``get_trial_data``: for each downsampled frame, + weight nearby acquisition lines by ``exp(-|dframe|/dt)`` and scatter their + superpixel line data into the ``(n_superpixels, n_ds_frames)`` accumulator. + + Parameters + ---------- + data_file : object + SLAP2 data-file-like object exposing ``header['linesPerCycle']``, + ``numCycles``, ``getLineData``, ``lineDataNumElements``, + ``lineSuperPixelIDs``, and ``lineFastZIdxs``. + ds_frames : ndarray + Downsampled-frame line grid from :func:`compute_ds_frames`. + dt : float + Downsample step, in lines. + super_pixel_ids : ndarray + Superpixel id lookup for this DMD, shape ``(n_superpixels, 1)``. + num_channels : int + Number of acquisition channels. + all_channels : bool + Read all channels (``getLineData(..., None)``) vs channel 1 only. + + Returns + ------- + dict + ``{"data", "data_count", "data2", "data_count2"}``. The second-channel + entries are ``None`` unless ``all_channels`` and ``num_channels >= 2``. + """ + lines_per_cycle = data_file.header["linesPerCycle"] + n_ds = len(ds_frames) + num_sp = super_pixel_ids.shape[0] + sp_flat = np.asarray(super_pixel_ids).reshape(-1) + read_second = all_channels and num_channels >= 2 + + time_windows, line_indices, cycle_indices = _build_time_windows( + ds_frames, dt, lines_per_cycle, data_file.numCycles + ) + all_lines, all_cycles, mapping = _build_line_cycle_cache( + line_indices, cycle_indices + ) + all_line_data = data_file.getLineData( + all_lines, all_cycles, 1 if not all_channels else None + ) + + acc = { + "data": np.zeros((num_sp, n_ds), dtype=np.float32), + "data_count": np.zeros((num_sp, n_ds), dtype=np.float32), + "data2": ( + np.zeros((num_sp, n_ds), dtype=np.float32) if read_second else None + ), + "data_count2": ( + np.zeros((num_sp, n_ds), dtype=np.float32) if read_second else None + ), + } + + for ds_ix in range(n_ds): + weights = np.exp(-np.abs(ds_frames[ds_ix] - time_windows[ds_ix]) / dt) + frame_lines = line_indices[ds_ix] + frame_cycles = cycle_indices[ds_ix] + for i, li in enumerate(frame_lines): + line_idx = int(li) + if data_file.lineDataNumElements[line_idx - 1] == 0: + continue + cache_idx = mapping[(line_idx, int(frame_cycles[i]))] + _accumulate_line( + acc, + all_line_data[cache_idx], + data_file.lineSuperPixelIDs[line_idx - 1], + data_file.lineFastZIdxs[line_idx - 1], + sp_flat, + weights[i], + ds_ix, + ) + return acc + + +def _open_slap2_file(path: str): # pragma: no cover - requires slap2_utils + """Open a SLAP2 data file (``MultiDataFiles`` for CYCLE globs else single). + + Parameters + ---------- + path : str + Absolute path to the SLAP2 ``.dat`` (or CYCLE-pattern) file. + + Returns + ------- + object + A ``slap2_utils`` data-file object. + """ + import importlib + + import slap2_utils + + importlib.reload(slap2_utils) + if re.search(r"CYCLE\d+", path): + return slap2_utils.MultiDataFiles(path) + return slap2_utils.DataFile(path) + + +def read_integration_trial_data( + trial_ix: int, + keep_trial: bool, + dmd_ix: int, + samp_freq: float, + super_pixel_ids: np.ndarray, + datadr: str, + trial_table: dict, + num_channels: int, + all_channels: bool = True, +) -> Optional[dict]: + """Read one integration trial's low-res superpixel activity. + + Opens the trial's SLAP2 file, builds the downsampled-frame grid, + accumulates weighted superpixel activity, and loads the matching alignment + data. Counts are returned unnormalized (the caller forms + ``data / data_count``); activity is divided by 100 to undo the acquisition + scaling. + + Parameters + ---------- + trial_ix : int + Trial index into ``trial_table`` columns. + keep_trial : bool + When False, the trial is skipped and ``None`` is returned. + dmd_ix : int + 0-based DMD index into ``trial_table`` rows. + samp_freq : float + Analysis/alignment sample rate, in Hz. + super_pixel_ids : ndarray + Superpixel id lookup for this DMD. + datadr : str + Raw-data directory holding the SLAP2 files. + trial_table : dict + Normalized trial table (``filename``, ``first_line``, ``last_line``, + ``fn_adata``). + num_channels : int + Number of acquisition channels. + all_channels : bool + Read all channels (default) vs channel 1 only. + + Returns + ------- + dict or None + ``{"data", "data_count", "alignment", "ds_frames", "data2", + "data_count2"}`` for kept trials, else ``None``. + """ + if not keep_trial: + return None + + source_fn = trial_table["filename"][dmd_ix, trial_ix] + first_line = trial_table["first_line"][dmd_ix, trial_ix] + last_line = trial_table["last_line"][dmd_ix, trial_ix] + + data_file = _open_slap2_file(os.path.join(datadr, source_fn)) + dt = 1.0 / samp_freq / data_file.metaData.linePeriod_s + ds_frames = compute_ds_frames(first_line, last_line, dt) + + acc = accumulate_superpixel_data( + data_file, ds_frames, dt, super_pixel_ids, num_channels, all_channels + ) + alignment = load_alignment_data_h5( + trial_table["fn_adata"][dmd_ix, trial_ix] + ) + + data2 = acc["data2"] + return { + "data": acc["data"] / 100.0, + "data_count": acc["data_count"], + "alignment": alignment, + "ds_frames": ds_frames, + "data2": None if data2 is None else data2 / 100.0, + "data_count2": acc["data_count2"], + } + + +def assemble_lowres_data(results: list, num_channels: int) -> dict: + """Concatenate per-trial reader results into low-res arrays. + + Port of the concatenation block that stitches per-trial outputs into the + per-DMD low-res dataset. ``None`` results (skipped trials) are dropped. + + Parameters + ---------- + results : list + Per-trial dicts from :func:`read_integration_trial_data` (or ``None``). + num_channels : int + Number of acquisition channels (>= 2 adds second-channel arrays). + + Returns + ------- + dict + ``lowResData``, ``lowResDataCt``, ``lowResMotionR/C/Z``, + ``lowResTrialID`` (and ``lowResData2``/``lowResDataCt2`` when + ``num_channels >= 2``). + """ + kept = [(i, r) for i, r in enumerate(results) if r is not None] + arrays = { + "lowResData": np.concatenate([r["data"] for _, r in kept], axis=1), + "lowResDataCt": np.concatenate( + [r["data_count"] for _, r in kept], axis=1 + ), + "lowResMotionR": np.concatenate( + [r["alignment"]["motionDSr"] for _, r in kept], axis=0 + ), + "lowResMotionC": np.concatenate( + [r["alignment"]["motionDSc"] for _, r in kept], axis=0 + ), + "lowResMotionZ": np.concatenate( + [r["alignment"]["motionDSz"] for _, r in kept], axis=0 + ), + "lowResTrialID": np.concatenate( + [np.ones_like(r["ds_frames"]) * i for i, r in kept], axis=0 + ), + } + if num_channels >= 2: + arrays["lowResData2"] = np.concatenate( + [r["data2"] for _, r in kept], axis=1 + ) + arrays["lowResDataCt2"] = np.concatenate( + [r["data_count2"] for _, r in kept], axis=1 + ) + return arrays + + +def save_lowres_data(path: str, arrays: dict) -> None: + """Save assembled low-res arrays to an ``.npz`` cache.""" + np.savez(path, **arrays) + + +def load_lowres_data(path: str, num_channels: int) -> dict: + """Load an ``.npz`` low-res cache into a dict of arrays. + + Parameters + ---------- + path : str + Path to the ``lowres_data_DMD{N}.npz`` cache. + num_channels : int + Number of acquisition channels (>= 2 also loads the second channel). + + Returns + ------- + dict + The cached low-res arrays. + """ + keys = [ + "lowResData", + "lowResDataCt", + "lowResMotionR", + "lowResMotionC", + "lowResMotionZ", + "lowResTrialID", + ] + if num_channels >= 2: + keys += ["lowResData2", "lowResDataCt2"] + with np.load(path) as data_arrays: + return {k: data_arrays[k] for k in keys} diff --git a/tests/test_bandsilo_trial_data.py b/tests/test_bandsilo_trial_data.py new file mode 100644 index 0000000..657991f --- /dev/null +++ b/tests/test_bandsilo_trial_data.py @@ -0,0 +1,280 @@ +"""Tests for giant_python.bandsilo.trial_data (pure + mocked-reader paths).""" + +import os +import tempfile +import unittest + +import numpy as np + +from giant_python.bandsilo import trial_data as td + + +class _MockDataFile: + """Minimal SLAP2 data-file stub for accumulate_superpixel_data. + + Models a single cycle with ``lines_per_cycle`` lines. Each line maps to a + superpixel id / fast-Z via ``line_super_pixel_ids`` and ``line_fastz`` and + returns a fixed per-channel value from ``get_line_data``. + """ + + def __init__(self, lines_per_cycle, num_cycles, line_specs, channels=2): + """Build a stub from per-line ``(sp_ids, fastz, values)`` specs.""" + self.header = {"linesPerCycle": lines_per_cycle} + self.numCycles = num_cycles + self._line_specs = line_specs + self._channels = channels + + class _Meta: + linePeriod_s = 1.0 + + self.metaData = _Meta() + self.lineDataNumElements = [len(spec[0]) for spec in line_specs] + self.lineSuperPixelIDs = [np.asarray(spec[0]) for spec in line_specs] + self.lineFastZIdxs = [np.asarray(spec[1]) for spec in line_specs] + + def getLineData(self, lines, cycles, channel): + """Return a per-line list of (n_positions, n_channels) arrays.""" + out = [] + for li in lines: + spec = self._line_specs[int(li) - 1] + values = np.asarray(spec[2], dtype=np.float32) + out.append(np.tile(values.reshape(-1, 1), (1, self._channels))) + return out + + +class TestNearestInterp(unittest.TestCase): + """nearest_interp mirrors MATLAB nearest interpolation.""" + + def test_single_point(self): + """A single sample returns yp unchanged.""" + yp = np.array([7.0]) + out = td.nearest_interp(np.array([0.0, 5.0]), np.array([2.0]), yp) + np.testing.assert_array_equal(out, yp) + + def test_midpoint_rounding(self): + """Query points snap to the nearest sample boundary.""" + xp = np.array([0.0, 10.0]) + yp = np.array([1.0, 2.0]) + out = td.nearest_interp(np.array([4.0, 6.0]), xp, yp) + np.testing.assert_array_equal(out, np.array([1.0, 2.0])) + + +class TestFastDilation(unittest.TestCase): + """fast_dilation covers the 3x3 fast path and the cv2 fallback.""" + + def test_3x3_fast_path(self): + """A single point dilates to its 8-neighborhood.""" + mask = np.zeros((5, 5), dtype=bool) + mask[2, 2] = True + out = td.fast_dilation(mask, np.ones((3, 3), np.uint8)) + self.assertEqual(out[1:4, 1:4].sum(), 9) + self.assertEqual(out.sum(), 9) + + def test_leading_axes_fast_path(self): + """The fast path dilates only the trailing two axes.""" + mask = np.zeros((2, 5, 5), dtype=bool) + mask[0, 2, 2] = True + out = td.fast_dilation(mask, np.ones((3, 3), np.uint8)) + self.assertEqual(out[0].sum(), 9) + self.assertEqual(out[1].sum(), 0) + + def test_generic_cv2_path(self): + """A non-3x3 kernel routes through the cv2 fallback.""" + mask = np.zeros((5, 5), dtype=bool) + mask[2, 2] = True + out = td.fast_dilation(mask, np.ones((1, 3), np.uint8)) + # 1x3 horizontal kernel dilates left/right only. + self.assertTrue(out[2, 1] and out[2, 3] and out[2, 2]) + self.assertFalse(out[1, 2] or out[3, 2]) + + def test_default_kernel(self): + """A None kernel uses the 7x7 default (generic path).""" + mask = np.zeros((9, 9), dtype=bool) + mask[4, 4] = True + out = td.fast_dilation(mask) + self.assertEqual(out[1:8, 1:8].sum(), 49) + + +class TestComputeDsFrames(unittest.TestCase): + """compute_ds_frames builds the downsampled line grid.""" + + def test_grid(self): + """The grid is ceil of arange(first, last+1, dt).""" + out = td.compute_ds_frames(1, 5, 2.0) + np.testing.assert_array_equal(out, np.array([1.0, 3.0, 5.0])) + + +class TestAccumulate(unittest.TestCase): + """accumulate_superpixel_data scatters weighted line data.""" + + def _super_pixel_ids(self): + """Two superpixels: sp1 z1 -> 101, sp2 z1 -> 201.""" + return np.array([[101], [201]], dtype=np.int32) + + def _data_file(self, channels=2): + """One cycle, two lines each mapping to one superpixel.""" + line_specs = [ + ([1], [1], [10.0]), # line 1 -> sp id 1*100+1 = 101, value 10 + ([2], [1], [20.0]), # line 2 -> sp id 2*100+1 = 201, value 20 + ] + return _MockDataFile(2, 1, line_specs, channels=channels) + + def test_single_channel(self): + """Channel-1 read populates data/data_count; no second channel.""" + df = self._data_file() + ds_frames = np.array([1.0, 2.0]) + acc = td.accumulate_superpixel_data( + df, ds_frames, 1.0, self._super_pixel_ids(), 1, False + ) + self.assertEqual(acc["data"].shape, (2, 2)) + self.assertIsNone(acc["data2"]) + # sp1 gets its strongest weight at frame 0 (line 1), sp2 at frame 1. + self.assertGreater(acc["data"][0, 0], 0) + self.assertGreater(acc["data"][1, 1], 0) + # Normalized value recovers the line value. + norm = acc["data"][0, 0] / acc["data_count"][0, 0] + self.assertAlmostEqual(norm, 10.0, places=4) + + def test_two_channels(self): + """all_channels + num_channels>=2 fills the second-channel arrays.""" + df = self._data_file() + ds_frames = np.array([1.0, 2.0]) + acc = td.accumulate_superpixel_data( + df, ds_frames, 1.0, self._super_pixel_ids(), 2, True + ) + self.assertIsNotNone(acc["data2"]) + self.assertEqual(acc["data2"].shape, (2, 2)) + self.assertGreater(acc["data_count2"][0, 0], 0) + + def test_zero_element_line_skipped(self): + """A line with zero data elements is skipped.""" + line_specs = [ + ([], [], []), # line 1: empty -> skipped + ([2], [1], [20.0]), + ] + df = _MockDataFile(2, 1, line_specs, channels=1) + acc = td.accumulate_superpixel_data( + df, np.array([1.0, 2.0]), 1.0, self._super_pixel_ids(), 1, False + ) + # sp1 (id 101) never accumulates because its only line is empty. + self.assertEqual(acc["data"][0].sum(), 0) + + +class TestReadIntegrationTrialData(unittest.TestCase): + """read_integration_trial_data orchestrates open+accumulate+alignment.""" + + def test_skipped_trial(self): + """keep_trial=False returns None without opening anything.""" + out = td.read_integration_trial_data( + 0, False, 0, 100.0, np.zeros((1, 1)), "/data", {}, 1 + ) + self.assertIsNone(out) + + def test_full_read_with_monkeypatched_open(self): + """A kept trial reads, scales by 1/100, and loads alignment.""" + sp_ids = np.array([[101], [201]], dtype=np.int32) + df = _MockDataFile( + 2, 1, [([1], [1], [10.0]), ([2], [1], [20.0])], channels=1 + ) + trial_table = { + "filename": np.array([["t1.dat"]], dtype=object), + "first_line": np.array([[1]]), + "last_line": np.array([[2]]), + "fn_adata": np.array([["a.h5"]], dtype=object), + } + + orig_open = td._open_slap2_file + orig_align = td.load_alignment_data_h5 + td._open_slap2_file = lambda path: df + td.load_alignment_data_h5 = lambda path: { + "motionDSr": np.zeros(2), + "motionDSc": np.zeros(2), + "motionDSz": np.zeros(2), + } + try: + out = td.read_integration_trial_data( + 0, + True, + 0, + 1.0, + sp_ids, + "/data", + trial_table, + 1, + all_channels=False, + ) + finally: + td._open_slap2_file = orig_open + td.load_alignment_data_h5 = orig_align + + self.assertIsNotNone(out) + self.assertEqual(out["data"].shape, (2, 2)) + self.assertIsNone(out["data2"]) + norm = out["data"][0, 0] / out["data_count"][0, 0] + self.assertAlmostEqual(norm, 10.0 / 100.0, places=6) + + +class TestAssembleAndCache(unittest.TestCase): + """assemble_lowres_data + save/load npz round trip.""" + + def _results(self, num_channels): + """Two synthetic trial results (one None) for assembly.""" + r0 = { + "data": np.ones((2, 3), dtype=np.float32), + "data_count": np.full((2, 3), 2.0, dtype=np.float32), + "alignment": { + "motionDSr": np.zeros(3), + "motionDSc": np.ones(3), + "motionDSz": np.full(3, 2.0), + }, + "ds_frames": np.arange(3), + "data2": np.full((2, 3), 5.0, dtype=np.float32), + "data_count2": np.full((2, 3), 2.0, dtype=np.float32), + } + r1 = { + "data": np.ones((2, 2), dtype=np.float32), + "data_count": np.full((2, 2), 2.0, dtype=np.float32), + "alignment": { + "motionDSr": np.zeros(2), + "motionDSc": np.ones(2), + "motionDSz": np.full(2, 2.0), + }, + "ds_frames": np.arange(2), + "data2": np.full((2, 2), 5.0, dtype=np.float32), + "data_count2": np.full((2, 2), 2.0, dtype=np.float32), + } + return [r0, None, r1] + + def test_assemble_single_channel(self): + """Single-channel assembly concatenates and builds trial IDs.""" + arrays = td.assemble_lowres_data(self._results(1), 1) + self.assertEqual(arrays["lowResData"].shape, (2, 5)) + self.assertNotIn("lowResData2", arrays) + # trial IDs: three 0s (r0) then two 2s (r1, original index 2). + np.testing.assert_array_equal( + arrays["lowResTrialID"], np.array([0, 0, 0, 2, 2]) + ) + + def test_assemble_two_channels(self): + """Two-channel assembly adds the second-channel arrays.""" + arrays = td.assemble_lowres_data(self._results(2), 2) + self.assertEqual(arrays["lowResData2"].shape, (2, 5)) + self.assertEqual(arrays["lowResDataCt2"].shape, (2, 5)) + + def test_save_load_round_trip(self): + """save_lowres_data + load_lowres_data preserve arrays.""" + arrays = td.assemble_lowres_data(self._results(2), 2) + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "lowres_data_DMD1.npz") + td.save_lowres_data(path, arrays) + loaded = td.load_lowres_data(path, 2) + np.testing.assert_array_equal( + loaded["lowResData"], arrays["lowResData"] + ) + np.testing.assert_array_equal( + loaded["lowResData2"], arrays["lowResData2"] + ) + + +if __name__ == "__main__": + unittest.main() From bc6e97ec36c0902f251c58b11f7241de933a6cc5 Mon Sep 17 00:00:00 2001 From: Michael Xie Date: Thu, 16 Jul 2026 12:48:53 -0700 Subject: [PATCH 05/55] add VS Code directory to .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 06a56dd..649f1f1 100644 --- a/.gitignore +++ b/.gitignore @@ -135,5 +135,8 @@ dmypy.json # option (not recommended) you can uncomment the following to ignore the entire idea folder. .idea/ +# VS Code +.vscode/ + # MacOs **/.DS_Store From ebb52dcf807da1d940007e1d0a0e1c12ecb28bee Mon Sep 17 00:00:00 2001 From: Michael Xie Date: Sun, 26 Jul 2026 07:50:50 -0700 Subject: [PATCH 06/55] implement activity image generation for bandscan --- src/giant_python/bandsilo/background.py | 1113 +++++++++++++++++++ src/giant_python/bandsilo/summary_images.py | 273 +++++ tests/test_bandsilo_background.py | 526 +++++++++ tests/test_bandsilo_summary_images.py | 164 +++ 4 files changed, 2076 insertions(+) create mode 100644 src/giant_python/bandsilo/background.py create mode 100644 src/giant_python/bandsilo/summary_images.py create mode 100644 tests/test_bandsilo_background.py create mode 100644 tests/test_bandsilo_summary_images.py diff --git a/src/giant_python/bandsilo/background.py b/src/giant_python/bandsilo/background.py new file mode 100644 index 0000000..38499c8 --- /dev/null +++ b/src/giant_python/bandsilo/background.py @@ -0,0 +1,1113 @@ +"""Motion binning, background/noise estimation, and rho computation. + +Ported from the background/noise block of +``extractSLAP2IntegrationSources.py`` (ref 1901-2356). This module owns the +kernels that turn the per-superpixel low-res traces into: + +* motion bins (3-D and 2-D) and the selected-pixel support, +* the PSF convolution matrices ``D``/``D_expanded`` and the sparse per-motion + ``H`` matrices, +* the interpolated background estimate (``build_interp_data`` + + rolling-window baseline) and its reassembly onto the superpixel grid, +* an affine noise-variance model producing ``data_std`` and the z-scored + ``residual``, +* the ``rho`` matched-filter response and its NaN-aware temporal smoothing. + +The summary images that consume these outputs (the mean image, and the +activity image built from ``rho``) live in +:mod:`giant_python.bandsilo.summary_images`. The full threading of these +kernels over the per-DMD arrays is done by the Phase-8 pipeline driver; here +each kernel is a standalone, testable function. + +The commented-out soft-impute background completion and rolling-MAD rho +normalization in the reference are intentionally dropped (dead code). +""" + +from __future__ import annotations + +from typing import List, Tuple + +import numpy as np +import scipy.ndimage as ndimage +import torch +from scipy import signal +from scipy.interpolate import RectBivariateSpline + + +def bin_motion( + motion_r: np.ndarray, motion_c: np.ndarray, motion_z: np.ndarray +) -> Tuple[np.ndarray, np.ndarray]: + """Group frames into unique rounded (row, col, z) motion bins. + + Parameters + ---------- + motion_r, motion_c, motion_z : ndarray + Per-frame downsampled motion in rows, columns, and z. + + Returns + ------- + unique_motion : ndarray of shape (n_bins, 3) + The unique rounded ``[row, col, z]`` motion vectors. + mot_inds : ndarray of shape (n_frames,) + Index of each frame's motion bin into ``unique_motion``. + """ + unique_motion, mot_inds = np.unique( + np.round(np.stack((motion_r, motion_c, motion_z), axis=1)), + axis=0, + return_inverse=True, + ) + # np.unique's inverse is 1-D on numpy<2 and 2.1+, but 2-D on 2.0; flatten + # so downstream bincount/isin behave identically across versions. + return unique_motion, np.reshape(mot_inds, -1) + + +def select_motion_bins( + unique_motion: np.ndarray, + mot_inds: np.ndarray, + motion_z: np.ndarray, + z_thresh: float = 1.5, + min_frames: int = 100, +) -> Tuple[np.ndarray, np.ndarray]: + """Keep motion bins near the median z with enough frames. + + A bin is kept when its z is within ``z_thresh`` of the median z (over all + frames) and it contains more than ``min_frames`` frames. After filtering, + the remaining frames are treated as having no z motion. + + Parameters + ---------- + unique_motion : ndarray of shape (n_bins, 3) + Unique motion vectors from :func:`bin_motion`. + mot_inds : ndarray of shape (n_frames,) + Per-frame motion-bin indices. + motion_z : ndarray + Per-frame z motion (used for the median). + z_thresh : float + Maximum absolute z deviation from the median, in microns. + min_frames : int + Minimum frame count (strictly greater) for a bin to be kept. + + Returns + ------- + mot_inds_to_keep : ndarray + Indices of the kept bins into ``unique_motion``. + frames_to_keep : ndarray of bool + Per-frame mask of frames belonging to a kept bin. + """ + median_z = np.median(motion_z) + bin_counts = np.bincount(mot_inds, minlength=unique_motion.shape[0]) + keep_mask = (np.abs(unique_motion[:, 2] - median_z) <= z_thresh) & ( + bin_counts > min_frames + ) + mot_inds_to_keep = np.nonzero(keep_mask)[0] + frames_to_keep = np.isin(mot_inds, mot_inds_to_keep) + return mot_inds_to_keep, frames_to_keep + + +def bin_motion_yx( + motion_r: np.ndarray, + motion_c: np.ndarray, + frames_to_keep: np.ndarray, +) -> Tuple[np.ndarray, np.ndarray]: + """Bin kept frames into unique rounded (row, col) motion bins. + + Parameters + ---------- + motion_r, motion_c : ndarray + Per-frame downsampled row/column motion. + frames_to_keep : ndarray of bool + Frames to bin; dropped frames get index ``-1``. + + Returns + ------- + unique_motion_to_keep_yx : ndarray of shape (n_yx_bins, 2) + Unique rounded ``[row, col]`` motion vectors over the kept frames. + mot_inds_yx : ndarray of shape (n_frames,), int32 + Per-frame bin index into ``unique_motion_to_keep_yx``; ``-1`` for + frames not kept. + """ + mot_inds_yx = -1 * np.ones((frames_to_keep.shape[0],), dtype=np.int32) + stacked = np.round( + np.stack((motion_r, motion_c), axis=1)[frames_to_keep, :] + ) + unique_motion_to_keep_yx, inv = np.unique( + stacked, axis=0, return_inverse=True + ) + mot_inds_yx[frames_to_keep] = np.reshape(inv, -1) + return unique_motion_to_keep_yx, mot_inds_yx + + +def build_selected_pixel_mask( + unique_motion_to_keep_yx: np.ndarray, + ref_d: np.ndarray, + ref_r: np.ndarray, + ref_c: np.ndarray, + num_fast_zs: int, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, + psf2d: np.ndarray, +) -> Tuple[np.ndarray, np.ndarray]: + """Build the selected-pixel support mask over all kept motion shifts. + + Marks every reference pixel shifted by each kept 2-D motion bin, then + dilates in-plane by the PSF footprint. + + Parameters + ---------- + unique_motion_to_keep_yx : ndarray of shape (n_yx_bins, 2) + Kept 2-D motion vectors. + ref_d, ref_r, ref_c : ndarray + Per-superpixel reference depth/row/column indices. + num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row : int + Grid geometry. + psf2d : ndarray + The (cropped) PSF for this DMD; its shape sets the dilation footprint. + + Returns + ------- + sel_pix_mask : ndarray of bool + ``(num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row)`` support. + sel_pix_idxs : ndarray of int + Flattened indices of the selected pixels. + """ + sel_pix_mask = np.zeros( + (num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row), dtype=bool + ) + for i in range(len(unique_motion_to_keep_yx)): + sel_pix_mask[ + ref_d, + ref_r + int(unique_motion_to_keep_yx[i, 0]), + ref_c + int(unique_motion_to_keep_yx[i, 1]), + ] = True + sel_pix_mask = ndimage.binary_dilation( + sel_pix_mask, + structure=np.ones((1, psf2d.shape[0], psf2d.shape[1]), dtype=bool), + ) + sel_pix_idxs = np.flatnonzero(sel_pix_mask) + return sel_pix_mask, sel_pix_idxs + + +def pixel_coords_from_idxs( + sel_pix_idxs: np.ndarray, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, +) -> np.ndarray: + """Convert flat selected-pixel indices to ``[z, row, col]`` coordinates. + + Parameters + ---------- + sel_pix_idxs : ndarray of int + Flattened selected-pixel indices. + dmd_pixels_per_column, dmd_pixels_per_row : int + Grid geometry. + + Returns + ------- + ndarray of shape (n_sel, 3), int32 + ``[z, row, col]`` per selected pixel. + """ + plane_size = dmd_pixels_per_column * dmd_pixels_per_row + pixel_coords = np.empty((len(sel_pix_idxs), 3), dtype=np.int32) + pixel_coords[:, 0] = sel_pix_idxs // plane_size + remainder = sel_pix_idxs % plane_size + pixel_coords[:, 1] = remainder // dmd_pixels_per_row + pixel_coords[:, 2] = remainder % dmd_pixels_per_row + return pixel_coords + + +def expand_psf( + psf2d: np.ndarray, ex_fac: int = 2 +) -> Tuple[torch.Tensor, Tuple[int, int], torch.Tensor, Tuple[int, int]]: + """Normalize a PSF and build a center-aligned higher-resolution copy. + + The expanded PSF is an ``ex_fac``x up-sampled cubic-spline interpolation + that keeps the center aligned, used to form the ``HD - HD_expanded`` + matched filter in :func:`compute_rho`. + + Parameters + ---------- + psf2d : ndarray + The (cropped) PSF for this DMD. + ex_fac : int + Expansion factor along each axis. + + Returns + ------- + psf_tensor : torch.Tensor + Sum-normalized PSF. + psf_center : tuple of int + ``(row, col)`` center of ``psf_tensor``. + psf_tensor_expanded : torch.Tensor + Sum-normalized expanded PSF. + psf_center_expanded : tuple of int + ``(row, col)`` center of ``psf_tensor_expanded``. + """ + psf_arr = np.asarray(psf2d, dtype=np.float32) + psf_tensor = torch.from_numpy(psf_arr).float() + psf_tensor = psf_tensor / torch.sum(psf_tensor) + psf_shape = psf_arr.shape + psf_center = (psf_shape[0] // 2, psf_shape[1] // 2) + + expanded_center_y = (psf_shape[0] * ex_fac - 1) / 2 + expanded_center_x = (psf_shape[1] * ex_fac - 1) / 2 + orig_y = torch.linspace( + -expanded_center_y, expanded_center_y, psf_shape[0] + ) + orig_x = torch.linspace( + -expanded_center_x, expanded_center_x, psf_shape[1] + ) + expanded_y = torch.linspace( + -expanded_center_y, expanded_center_y, psf_shape[0] * ex_fac + ) + expanded_x = torch.linspace( + -expanded_center_x, expanded_center_x, psf_shape[1] * ex_fac + ) + interp_spline = RectBivariateSpline( + orig_y.numpy(), orig_x.numpy(), psf_tensor.numpy() + ) + psf_tensor_expanded = torch.from_numpy( + interp_spline(expanded_y.numpy(), expanded_x.numpy()) + ).float() + psf_tensor_expanded = psf_tensor_expanded / torch.sum(psf_tensor_expanded) + psf_shape_expanded = psf_tensor_expanded.shape + psf_center_expanded = ( + psf_shape_expanded[0] // 2, + psf_shape_expanded[1] // 2, + ) + return psf_tensor, psf_center, psf_tensor_expanded, psf_center_expanded + + +def selected_pixels_2d_for_plane( + sel_pix_idxs: np.ndarray, + plane_z: int, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, +) -> Tuple[np.ndarray, np.ndarray]: + """Return the in-plane ``[row, col]`` coords of one z-plane's pixels. + + Parameters + ---------- + sel_pix_idxs : ndarray of int + Flattened selected-pixel indices (all planes). + plane_z : int + The z-plane to select. + dmd_pixels_per_column, dmd_pixels_per_row : int + Grid geometry. + + Returns + ------- + z_idxs : ndarray + Positions (into ``sel_pix_idxs``) of pixels in ``plane_z``. + sel_pixels_2d : ndarray of shape (n_plane, 2) + ``[row, col]`` coordinates of those pixels. + """ + plane_size = dmd_pixels_per_column * dmd_pixels_per_row + sel_pix_z = sel_pix_idxs // plane_size + sel_pix_remainder = sel_pix_idxs % plane_size + z_idxs = np.flatnonzero(sel_pix_z == plane_z) + sel_pixels_2d = np.column_stack( + [ + sel_pix_remainder[z_idxs] // dmd_pixels_per_row, + sel_pix_remainder[z_idxs] % dmd_pixels_per_row, + ] + ) + return z_idxs, sel_pixels_2d + + +def build_convolution_matrix( + sel_pixels_2d: np.ndarray, + psf_tensor: torch.Tensor, + psf_center: Tuple[int, int], +) -> torch.Tensor: + """Build a dense PSF-convolution matrix for one plane's selected pixels. + + Entry ``(t, s)`` is the PSF weight for the offset from source pixel ``s`` + to target pixel ``t`` (zero where the offset falls outside the PSF). Used + with both the base and expanded PSFs to form ``D``/``D_expanded``. + + Parameters + ---------- + sel_pixels_2d : ndarray of shape (n_plane, 2) + In-plane ``[row, col]`` coordinates. + psf_tensor : torch.Tensor + Sum-normalized PSF (base or expanded). + psf_center : tuple of int + ``(row, col)`` center of ``psf_tensor``. + + Returns + ------- + torch.Tensor + Dense ``(n_plane, n_plane)`` convolution matrix. + """ + n = sel_pixels_2d.shape[0] + if n == 0: + return torch.zeros((0, 0), dtype=torch.float32) + src_rows = sel_pixels_2d[:, 0][np.newaxis, :] + src_cols = sel_pixels_2d[:, 1][np.newaxis, :] + tgt_rows = sel_pixels_2d[:, 0][:, np.newaxis] + tgt_cols = sel_pixels_2d[:, 1][:, np.newaxis] + rel_rows = tgt_rows - src_rows + psf_center[0] + rel_cols = tgt_cols - src_cols + psf_center[1] + psf_np = psf_tensor.numpy() + psf_shape = psf_np.shape + valid_mask = ( + (rel_rows >= 0) + & (rel_rows < psf_shape[0]) + & (rel_cols >= 0) + & (rel_cols < psf_shape[1]) + ) + out = np.zeros((n, n), dtype=np.float32) + out[valid_mask] = psf_np[rel_rows[valid_mask], rel_cols[valid_mask]] + return torch.from_numpy(out) + + +def _collect_interp_columns( + cols, s_c, s_r, sel_idxs_by_col_all, sel_rows, r0, r1 +): + """Group a motion bin's reference pixels by column for interpolation. + + Columns with a single reference pixel become ``single_points`` (copied + verbatim); columns with several become sorted ``(rows, source-index)`` + entries for linear interpolation. + """ + rows_by_col = {} + idxs_by_col = {} + single_points = [] + for c in cols: + c_int = int(c) + if c_int not in sel_idxs_by_col_all: + continue + data_ix = np.flatnonzero(s_c == c) + if data_ix.size == 1: + rr = int(s_r[data_ix[0]]) + col_sel = sel_idxs_by_col_all[c_int] + rr_match = col_sel[sel_rows[col_sel] == rr] + if rr_match.size > 0 and (r0 <= rr <= r1): + single_points.append((rr_match, int(data_ix[0]))) + continue + rows = s_r[data_ix] + order = rows.argsort(kind="mergesort") + rows_by_col[c_int] = rows[order] + idxs_by_col[c_int] = data_ix[order] + return rows_by_col, idxs_by_col, single_points + + +def _interp_columns( + out, + rows_by_col, + idxs_by_col, + sel_idxs_by_col_all, + sel_rows, + data, + frames, + dtype, +): + """Linearly interpolate each multi-point column onto its selected pixels. + + Selected rows that coincide with a reference row are copied exactly; those + strictly between two reference rows are linearly interpolated; those + outside the sampled row range are left untouched (NaN). + """ + for c_int, rows in rows_by_col.items(): + col_sel_pix_idxs = sel_idxs_by_col_all[c_int] + target_rows = sel_rows[col_sel_pix_idxs] + src_idx = idxs_by_col[c_int] + + pos = np.searchsorted(rows, target_rows, side="left") + in_bounds = pos < rows.size + exact = np.zeros_like(pos, dtype=bool) + exact[in_bounds] = rows[pos[in_bounds]] == target_rows[in_bounds] + + if np.any(exact): + exact_rows = np.flatnonzero(exact) + exact_src = src_idx[pos[exact]] + out[np.ix_(col_sel_pix_idxs[exact_rows], frames)] = data[ + exact_src + ][:, frames] + + lo = pos - 1 + hi = pos + interp_mask = (~exact) & (lo >= 0) & (hi < rows.size) + if not np.any(interp_mask): + continue + + interp_rows = np.flatnonzero(interp_mask) + lo_v = lo[interp_mask] + hi_v = hi[interp_mask] + row_lo = rows[lo_v].astype(dtype, copy=False) + row_hi = rows[hi_v].astype(dtype, copy=False) + denom = row_hi - row_lo + nonzero = denom != 0 + if not np.any(nonzero): + # Unreachable defensive guard (mirrors the reference): sorted + # distinct rows make searchsorted never straddle equal rows, so + # denom is always non-zero here. + continue # pragma: no cover + + interp_rows = interp_rows[nonzero] + lo_v = lo_v[nonzero] + hi_v = hi_v[nonzero] + row_lo = row_lo[nonzero] + row_hi = row_hi[nonzero] + alpha = ( + (target_rows[interp_rows].astype(dtype) - row_lo) + / (row_hi - row_lo) + )[:, None] + + vals_lo = data[src_idx[lo_v]][:, frames] + vals_hi = data[src_idx[hi_v]][:, frames] + out[np.ix_(col_sel_pix_idxs[interp_rows], frames)] = ( + 1.0 - alpha + ) * vals_lo + alpha * vals_hi + + +def build_interp_data( + data: np.ndarray, + ref_r: np.ndarray, + ref_c: np.ndarray, + sel_pixels_2d: np.ndarray, + unique_motion_to_keep_yx: np.ndarray, + mot_inds_yx: np.ndarray, + height: int = 800, + width: int = 1280, + dtype=np.float32, +) -> Tuple[np.ndarray, Tuple[int, int], Tuple[int, int]]: + """Interpolate reference-pixel traces onto the selected-pixel grid. + + For each kept motion bin, the reference pixels (shifted by the bin's + motion) are linearly interpolated along rows, per column, onto the selected + pixels of the plane, filling one background estimate per selected pixel and + frame. Selected pixels outside the sampled row range remain NaN. + + Parameters + ---------- + data : ndarray of shape (n_ref, n_frames) + Reference-pixel traces for one plane. + ref_r, ref_c : ndarray + Reference-pixel row/column indices for this plane. + sel_pixels_2d : ndarray of shape (n_sel, 2) + Selected-pixel ``[row, col]`` coordinates for this plane. + unique_motion_to_keep_yx : ndarray of shape (n_yx_bins, 2) + Kept 2-D motion vectors. + mot_inds_yx : ndarray of shape (n_frames,) + Per-frame motion-bin index (``-1`` for dropped frames). + height, width : int + Sensor row/column extents used to window valid shifts. + dtype : numpy dtype + Working/output dtype. + + Returns + ------- + out : ndarray of shape (n_sel, n_frames) + Interpolated background estimate (NaN where unsampled). + row_bounds : tuple of int + ``(r0, r1)`` inclusive row window. + col_bounds : tuple of int + ``(c0, c1)`` inclusive column window. + """ + ref_r = np.asarray(ref_r, dtype=np.int32) + ref_c = np.asarray(ref_c, dtype=np.int32) + data = np.asarray(data, dtype=dtype) + + r0 = int(max(0, ref_r.min()) + unique_motion_to_keep_yx[:, 0].min()) + r1 = int(min(height, ref_r.max()) + unique_motion_to_keep_yx[:, 0].max()) + c0 = int(max(0, ref_c.min()) + unique_motion_to_keep_yx[:, 1].min()) + c1 = int(min(width, ref_c.max()) + unique_motion_to_keep_yx[:, 1].max()) + + out = np.full((sel_pixels_2d.shape[0], data.shape[1]), np.nan, dtype=dtype) + + sel_cols = sel_pixels_2d[:, 1] + sel_rows = sel_pixels_2d[:, 0] + unique_sel_cols = np.unique(sel_cols) + sel_idxs_by_col_all = { + int(c): np.flatnonzero(sel_cols == c) for c in unique_sel_cols + } + + frames_by_motion = [ + np.flatnonzero(mot_inds_yx == idx) + for idx in range(len(unique_motion_to_keep_yx)) + ] + + for m_idx, frames in enumerate(frames_by_motion): + if frames.size == 0: + continue + + s_r = ref_r + int(unique_motion_to_keep_yx[m_idx, 0]) + s_c = ref_c + int(unique_motion_to_keep_yx[m_idx, 1]) + + in_win = (s_r >= r0) & (s_r <= r1) & (s_c >= c0) & (s_c <= c1) + if not np.any(in_win): + continue + cols = np.unique(s_c[in_win]) + + rows_by_col, idxs_by_col, single_points = _collect_interp_columns( + cols, s_c, s_r, sel_idxs_by_col_all, sel_rows, r0, r1 + ) + _interp_columns( + out, + rows_by_col, + idxs_by_col, + sel_idxs_by_col_all, + sel_rows, + data, + frames, + dtype, + ) + for rr_match, src_pos in single_points: + out[np.ix_(rr_match, frames)] = data[src_pos, frames] + + return out, (r0, r1), (c0, c1) + + +def baseline_window_frames(align_hz: float, baseline_window_s: float) -> int: + """Return the rolling-baseline window length in frames. + + Parameters + ---------- + align_hz : float + Alignment/analysis rate, in Hz. + baseline_window_s : float + Baseline window, in seconds. + + Returns + ------- + int + ``int(align_hz * baseline_window_s)``. + """ + return int(align_hz * baseline_window_s) + + +def compute_rolling_baseline( + interp_data: np.ndarray, baseline_window: int +) -> np.ndarray: + """Compute a NaN-aware rolling-mean baseline of the interpolated data. + + Uses running window sums of values and of the valid-sample count, then + divides, so NaNs contribute nothing and windows with no valid samples stay + NaN. ``interp_data`` is modified in place (NaNs replaced by 0). + + Parameters + ---------- + interp_data : ndarray of shape (n_sel, n_frames) + Interpolated background estimate (from :func:`build_interp_data`). + baseline_window : int + Rolling window length, in frames. + + Returns + ------- + ndarray + The rolling-mean background estimate (NaN where no valid samples). + """ + valid = ~np.isnan(interp_data) + np.nan_to_num(interp_data, copy=False, nan=0.0) + + sum_vals = ( + ndimage.uniform_filter1d( + interp_data, size=baseline_window, axis=1, mode="nearest" + ) + * baseline_window + ) + valid_f = valid.astype(np.float32, copy=False) + count_vals = ( + ndimage.uniform_filter1d( + valid_f, size=baseline_window, axis=1, mode="nearest" + ) + * baseline_window + ) + + interp_data_background = np.empty_like(sum_vals, dtype=interp_data.dtype) + interp_data_background.fill(np.nan) + np.divide( + sum_vals, count_vals, out=interp_data_background, where=count_vals > 0 + ) + return interp_data_background + + +def assemble_background( + interp_data_background: np.ndarray, + unique_motion_to_keep_yx: np.ndarray, + mot_inds_yx: np.ndarray, + sel_pix_idxs: np.ndarray, + ref_d: np.ndarray, + ref_r: np.ndarray, + ref_c: np.ndarray, + n_super_pixels: int, + n_frames: int, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, +) -> np.ndarray: + """Map the selected-pixel background back onto superpixels per motion bin. + + For each kept motion bin and its frames, each superpixel's motion-shifted + reference pixel is looked up in ``sel_pix_idxs`` and its background trace + is copied onto that superpixel. + + Parameters + ---------- + interp_data_background : ndarray of shape (n_sel, n_frames) + Rolling-baseline background on the selected-pixel grid. + unique_motion_to_keep_yx : ndarray of shape (n_yx_bins, 2) + Kept 2-D motion vectors. + mot_inds_yx : ndarray of shape (n_frames,) + Per-frame motion-bin index (``-1`` for dropped frames). + sel_pix_idxs : ndarray of int + Sorted flat selected-pixel indices. + ref_d, ref_r, ref_c : ndarray + Per-superpixel reference depth/row/column indices. + n_super_pixels, n_frames : int + Output shape. + dmd_pixels_per_column, dmd_pixels_per_row : int + Grid geometry. + + Returns + ------- + ndarray of shape (n_super_pixels, n_frames), float32 + Per-superpixel background (NaN where unmapped). + """ + plane_size = dmd_pixels_per_column * dmd_pixels_per_row + background = np.full((n_super_pixels, n_frames), np.nan, dtype=np.float32) + + for motion_idx in range(len(unique_motion_to_keep_yx)): + motion_frames = np.flatnonzero(mot_inds_yx == motion_idx) + d_r = int(unique_motion_to_keep_yx[motion_idx, 0]) + d_c = int(unique_motion_to_keep_yx[motion_idx, 1]) + s_d = ref_d + s_r = ref_r + d_r + s_c = ref_c + d_c + + shifted_indices = s_d * plane_size + s_r * dmd_pixels_per_row + s_c + + bg_idxs = np.searchsorted(sel_pix_idxs, shifted_indices) + idxs_mask = bg_idxs < sel_pix_idxs.size + idxs_mask[idxs_mask] &= ( + sel_pix_idxs[bg_idxs[idxs_mask]] == shifted_indices[idxs_mask] + ) + + if np.any(idxs_mask): + valid_rows = np.flatnonzero(idxs_mask) + background[ + np.ix_(valid_rows, motion_frames) + ] = interp_data_background[ + np.ix_(bg_idxs[valid_rows], motion_frames) + ] + + return background + + +def fit_noise_variance_model( + low_res_data_norm: np.ndarray, + background: np.ndarray, + v_im: np.ndarray, + vif: float = 1.38, +) -> Tuple[np.ndarray, float, float]: + """Calibrate an affine noise-variance model and return per-sample std. + + Fits ``Var ~= Vk * (background * vIM) + Vb`` from the brightest pixels over + the first valid frames, then returns the predicted standard deviation for + every sample. + + Parameters + ---------- + low_res_data_norm : ndarray of shape (n_superpixels, n_frames) + Count-normalized channel-1 traces. + background : ndarray of shape (n_superpixels, n_frames) + Per-superpixel background estimate. + v_im : ndarray of shape (n_superpixels, n_frames) + Inverse count image (``1 / lowResDataCt``). + vif : float + Variance inflation factor. + + Returns + ------- + data_std : ndarray + Per-sample predicted standard deviation. + v_k : float + Fitted multiplicative variance coefficient. + v_b : float + Fitted additive variance floor. + """ + first_valid_frames = np.flatnonzero( + np.any(~np.isnan(background[:, :1000]), axis=0) + ) + var_im = np.nanvar(low_res_data_norm[:, first_valid_frames], axis=1) + v_b = np.nanpercentile(var_im, 5) * vif + var_pred = np.nanmean( + background[:, first_valid_frames], axis=1 + ) * np.nanmean(v_im[:, first_valid_frames], axis=1) + sel_bright = var_pred > np.nanpercentile(var_pred, 90) + v_k = np.nanpercentile( + (var_im[sel_bright] - (v_b / vif)) / var_pred[sel_bright], 10 + ) + data_std = np.sqrt(np.clip(v_k * background * v_im, 0, None) + v_b) + return data_std, v_k, v_b + + +def compute_residual( + low_res_data_norm: np.ndarray, + background: np.ndarray, + data_std: np.ndarray, +) -> np.ndarray: + """Return the background-subtracted, noise-normalized residual. + + Parameters + ---------- + low_res_data_norm : ndarray + Count-normalized traces. + background : ndarray + Per-superpixel background estimate. + data_std : ndarray + Per-sample predicted standard deviation. + + Returns + ------- + ndarray + ``(low_res_data_norm - background) / data_std``. + """ + return (low_res_data_norm - background) / data_std + + +def build_motion_h_matrices( + sparse_h_inds: np.ndarray, + sparse_h_vals: np.ndarray, + unique_motion_to_keep_yx: np.ndarray, + sel_pix_idxs: np.ndarray, + num_super_pixels: int, + dmd_pixels_per_row: int, +) -> List[torch.Tensor]: + """Build one sparse superpixel<-selected-pixel ``H`` per motion bin. + + Each motion bin shifts the base ``H`` columns (image pixels) by its 2-D + motion, then remaps them into selected-pixel column space. + + Parameters + ---------- + sparse_h_inds : ndarray of shape (2, nnz) + Base ``H`` COO row/column indices (from ``geometry.build_sparse_h``). + sparse_h_vals : ndarray of shape (nnz,) + Base ``H`` values. + unique_motion_to_keep_yx : ndarray of shape (n_yx_bins, 2) + Kept 2-D motion vectors. + sel_pix_idxs : ndarray of int + Sorted flat selected-pixel indices. + num_super_pixels : int + Number of superpixels (``H`` row count). + dmd_pixels_per_row : int + Grid geometry (column-shift stride). + + Returns + ------- + list of torch.Tensor + One sparse COO ``(num_super_pixels, n_sel)`` tensor per motion bin. + """ + n_mot = len(unique_motion_to_keep_yx) + h_mots: List[torch.Tensor] = [None] * n_mot + base_sparse_rows = sparse_h_inds[0] + base_sparse_cols = sparse_h_inds[1] + motion_shifts = unique_motion_to_keep_yx[:, 0].astype( + np.int64, copy=False + ) * dmd_pixels_per_row + unique_motion_to_keep_yx[:, 1].astype( + np.int64, copy=False + ) + shifted_inds = np.empty((2, base_sparse_cols.shape[0]), dtype=np.int64) + shifted_inds[0] = base_sparse_rows + for i, pix_shift in enumerate(motion_shifts): + shifted_cols = base_sparse_cols + pix_shift + shifted_inds[1] = np.searchsorted(sel_pix_idxs, shifted_cols) + h_mots[i] = torch.sparse_coo_tensor( + shifted_inds, + sparse_h_vals, + (num_super_pixels, sel_pix_idxs.shape[0]), + dtype=torch.float32, + ) + return h_mots + + +def _plane_column_maps( + sel_pix_idxs: np.ndarray, num_fast_zs: int, plane_size: int +) -> Tuple[list, list, list]: + """Precompute per-z column masks, indices, and remaps into local space. + + Returns three lists (one entry per z): the boolean column mask, the column + indices of that plane's selected pixels, and a remap from global to local + (per-plane) column index (``-1`` outside the plane). + """ + sel_pix_z = sel_pix_idxs // plane_size + z_masks = [torch.from_numpy(sel_pix_z == z) for z in range(num_fast_zs)] + z_col_idxs = [torch.nonzero(m, as_tuple=False).squeeze(1) for m in z_masks] + z_remaps = [] + for z in range(num_fast_zs): + remap = torch.full((sel_pix_idxs.shape[0],), -1, dtype=torch.long) + if z_col_idxs[z].numel() > 0: + remap[z_col_idxs[z]] = torch.arange( + z_col_idxs[z].numel(), dtype=torch.long + ) + z_remaps.append(remap) + return z_masks, z_col_idxs, z_remaps + + +def _valid_sel_cols_for_motion( + motion_idx: int, + unique_motion_to_keep_yx: np.ndarray, + ref_d: np.ndarray, + ref_r: np.ndarray, + ref_c: np.ndarray, + sel_pix_idxs: np.ndarray, + num_fast_zs: int, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, + psf2d: np.ndarray, +) -> np.ndarray: + """Return a per-selected-pixel mask of columns valid for one motion bin. + + The valid region is the PSF-dilated reference support minus its + horizontally-eroded border (erosion width ``2 * psf_w - 1``), evaluated at + this bin's motion shift. + """ + plane_size = dmd_pixels_per_column * dmd_pixels_per_row + valid_pix_mask = np.zeros( + (num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row), dtype=bool + ) + valid_pix_mask[ + ref_d, + ref_r + int(unique_motion_to_keep_yx[motion_idx, 0]), + ref_c + int(unique_motion_to_keep_yx[motion_idx, 1]), + ] = True + valid_pix_mask = ndimage.binary_dilation( + valid_pix_mask, + structure=np.ones((1, psf2d.shape[0], psf2d.shape[1]), dtype=bool), + ) + valid_pix_mask = ndimage.binary_erosion( + valid_pix_mask, + structure=np.ones( + (1, psf2d.shape[0], psf2d.shape[1] * 2 - 1), dtype=bool % TODO: check if dimensions here should change + ), + ) + valid_pix_idxs = np.flatnonzero(valid_pix_mask) + valid_lookup = np.zeros(num_fast_zs * plane_size, dtype=bool) + valid_lookup[valid_pix_idxs] = True + return valid_lookup[sel_pix_idxs] + + +def compute_rho( + residual: np.ndarray, + mot_inds_yx: np.ndarray, + unique_motion_to_keep_yx: np.ndarray, + h_mots: List[torch.Tensor], + d_mats: List[torch.Tensor], + d_mats_expanded: List[torch.Tensor], + sel_pix_idxs: np.ndarray, + ref_d: np.ndarray, + ref_r: np.ndarray, + ref_c: np.ndarray, + num_fast_zs: int, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, + psf2d: np.ndarray, +) -> np.ndarray: + """Compute the ``rho`` matched-filter response on the selected pixels. + + For each motion bin and z-plane, restricts ``H`` to that plane, forms the + normalized ``HD`` and ``HD_expanded`` projections, and projects the + residual onto ``HD - HD_expanded`` over the plane's valid columns. + + Parameters + ---------- + residual : ndarray of shape (n_superpixels, n_frames) + Noise-normalized residual from :func:`compute_residual`. + mot_inds_yx : ndarray of shape (n_frames,) + Per-frame motion-bin index (``-1`` for dropped frames). + unique_motion_to_keep_yx : ndarray of shape (n_yx_bins, 2) + Kept 2-D motion vectors. + h_mots : list of torch.Tensor + Per-motion sparse ``H`` from :func:`build_motion_h_matrices`. + d_mats, d_mats_expanded : list of torch.Tensor + Per-plane base/expanded convolution matrices from + :func:`build_convolution_matrix`. + sel_pix_idxs : ndarray of int + Sorted flat selected-pixel indices. + ref_d, ref_r, ref_c : ndarray + Per-superpixel reference depth/row/column indices. + num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row : int + Grid geometry. + psf2d : ndarray + The (cropped) PSF for this DMD. + + Returns + ------- + ndarray of shape (n_sel, n_frames), float32 + The ``rho`` response (NaN where not computed). + """ + plane_size = dmd_pixels_per_column * dmd_pixels_per_row + z_masks, z_col_idxs, z_remaps = _plane_column_maps( + sel_pix_idxs, num_fast_zs, plane_size + ) + z_col_idxs_np = [zi.numpy() for zi in z_col_idxs] + + rho = np.full( + (len(sel_pix_idxs), residual.shape[1]), np.nan, dtype=np.float32 + ) + + for i in range(len(unique_motion_to_keep_yx)): + motion_frames = np.flatnonzero(mot_inds_yx == i) + if motion_frames.size == 0: + continue + + h_mot = h_mots[i].coalesce() + h_idxs = h_mot.indices() + h_vals = h_mot.values() + nrows, _ = h_mot.size() + residual_motion = residual[:, motion_frames].T + + valid_sel_cols = _valid_sel_cols_for_motion( + i, + unique_motion_to_keep_yx, + ref_d, + ref_r, + ref_c, + sel_pix_idxs, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + psf2d, + ) + + for z in range(num_fast_zs): + new_ncols = int(z_col_idxs[z].numel()) + if new_ncols == 0: + continue + + z_mask = z_masks[z] + keep_mask = z_mask[h_idxs[1]] + if keep_mask.sum().item() == 0: + # Unreachable defensive guard (mirrors the reference): a plane + # with selected pixels (new_ncols > 0) always has H columns in + # it, since both derive from the same in-plane PSF footprint. + continue # pragma: no cover + + remap = z_remaps[z] + new_rows = h_idxs[0, keep_mask] + new_cols = remap[h_idxs[1, keep_mask]] + new_idxs = torch.stack([new_rows, new_cols], dim=0) + new_vals = h_vals[keep_mask] + h_sub = torch.sparse_coo_tensor( + new_idxs, new_vals, (nrows, new_ncols), dtype=torch.float32 + ).coalesce() + + hd = torch.sparse.mm(h_sub, d_mats[z]) + hd = hd / torch.sum(hd, dim=0, keepdim=True) + hd_expanded = torch.sparse.mm(h_sub, d_mats_expanded[z]) + hd_expanded = hd_expanded / torch.sum( + hd_expanded, dim=0, keepdim=True + ) + hd_diff = hd - hd_expanded + + z_cols = z_col_idxs_np[z] + z_valid_mask = valid_sel_cols[z_cols] + if not np.any(z_valid_mask): + continue + z_valid_cols = z_cols[z_valid_mask] + local_valid_cols = remap[torch.from_numpy(z_valid_cols)].long() + + rho[np.ix_(z_valid_cols, motion_frames)] = ( + residual_motion @ hd_diff[:, local_valid_cols].numpy() + ).T + + return rho + + +def mask_high_nan_rho(rho: np.ndarray, thresh: float = 0.5) -> np.ndarray: + """NaN-out rows of ``rho`` that are mostly NaN and return the NaN fraction. + + ``rho`` is modified in place. Rows whose NaN fraction exceeds ``thresh`` + are set entirely to NaN. + + Parameters + ---------- + rho : ndarray of shape (n_sel, n_frames) + The rho response. + thresh : float + NaN-fraction threshold above which a row is dropped. + + Returns + ------- + ndarray of shape (n_sel,) + Per-row NaN fraction (before dropping). + """ + nan_ct = np.mean(np.isnan(rho), axis=1) + rho[nan_ct > thresh] = np.nan + return nan_ct + + +def decay_kernel_1d(decay_tau_s: float, align_hz: float) -> np.ndarray: + """Build the normalized causal exponential-decay smoothing kernel. + + Parameters + ---------- + decay_tau_s : float + Calcium-decay time constant, in seconds. + align_hz : float + Alignment/analysis rate, in Hz. + + Returns + ------- + ndarray + A 1-D kernel spanning ``3 * tau`` frames, summing to 1. + """ + decay_tau_frames = decay_tau_s * align_hz + k1d = np.exp( + np.linspace( + -np.ceil(decay_tau_frames * 3), + 0, + int(np.ceil(decay_tau_frames * 3) + 1), + ) + / decay_tau_frames + ) + return k1d / np.sum(k1d) + + +def smooth_rho(rho: np.ndarray, k1d: np.ndarray) -> np.ndarray: + """NaN-aware temporal smoothing of ``rho`` with a 1-D decay kernel. + + Convolves along time only, dividing the smoothed values by the smoothed + valid-sample count and NaN-ing samples whose effective count drops below + ``0.75``. Rows are processed in chunks to bound temporary memory. + ``rho`` is modified in place and also returned. + + Parameters + ---------- + rho : ndarray of shape (n_sel, n_frames) + The rho response (NaN allowed). + k1d : ndarray + 1-D smoothing kernel from :func:`decay_kernel_1d`. + + Returns + ------- + ndarray + The smoothed ``rho``. + """ + k2d = np.expand_dims(k1d, 0) + n_time = rho.shape[1] + bytes_per_row = max(1, n_time) * np.dtype(np.float32).itemsize * 3 + row_chunk = max(64, min(4096, int(200_000_000 // bytes_per_row))) + + for r0 in range(0, rho.shape[0], row_chunk): + r1 = min(r0 + row_chunk, rho.shape[0]) + rc = rho[r0:r1].copy() + row_has_data = np.any(np.isfinite(rc), axis=1) + if not np.any(row_has_data): + rho[r0:r1] = rc + continue + + rc_valid = rc[row_has_data] + rho_num = np.nan_to_num(rc_valid, nan=0.0) + rho_den = signal.convolve( + np.isfinite(rc_valid).astype(np.float32), k2d, mode="same" + ) + rho_num = signal.convolve(rho_num, k2d, mode="same") + valid_den = rho_den > 0.75 + np.divide(rho_num, rho_den, out=rho_num, where=valid_den) + rho_num[~valid_den] = np.nan + rc[row_has_data] = rho_num + rho[r0:r1] = rc + + return rho diff --git a/src/giant_python/bandsilo/summary_images.py b/src/giant_python/bandsilo/summary_images.py new file mode 100644 index 0000000..a163a36 --- /dev/null +++ b/src/giant_python/bandsilo/summary_images.py @@ -0,0 +1,273 @@ +"""Summary/visualization images: the mean image and the activity image. + +Ported from ``extractSLAP2IntegrationSources.py``. Groups the two per-DMD +summary images written to ``experiment_summary.h5`` / consumed downstream: + +* :func:`compute_mean_image` (ref 1913-1918) scatters the kept-frame mean + trace onto the reference-pixel grid, producing the ``mean_im`` visualization. +* :func:`accumulate_activity_image` / :func:`finalize_activity_image` (ref + 2358-2504) turn the smoothed ``rho`` from + :mod:`giant_python.bandsilo.background` into the ``act_im`` activity image + that Phase-5 peak detection localizes sources in. + +The activity image is built by sweeping ``rho`` in temporal batches, finding +spatio-temporal local maxima (a voxel greater than its 6 spatial neighbors in +the same frame and its temporal neighbors), and accumulating their squared +value; it is then masked where data was never valid and has a local +(1x11x11) NaN-median subtracted. + +The reference's optional ``profile_activity_map`` timing instrumentation is +intentionally dropped (it only prints timings and does not affect output). +""" + +from __future__ import annotations + +import numpy as np +import scipy.ndimage as ndimage + +from .trial_data import fast_dilation + + +def compute_mean_image( + low_res_data_norm: np.ndarray, + unique_motion: np.ndarray, + mot_inds: np.ndarray, + frames_to_keep: np.ndarray, + ref_d: np.ndarray, + ref_r: np.ndarray, + ref_c: np.ndarray, + num_fast_zs: int, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, + num_channels: int, + low_res_data2_norm: np.ndarray = None, +) -> np.ndarray: + """Scatter the kept-frame mean trace onto the reference pixel grid. + + The mean over kept frames is placed at each superpixel's reference pixel, + shifted by the most common motion bin, producing the ``mean_im`` + visualization ``(channels, z, rows, cols)``. + + Parameters + ---------- + low_res_data_norm : ndarray of shape (n_superpixels, n_frames) + Count-normalized channel-1 low-res traces. + unique_motion : ndarray of shape (n_bins, 3) + Unique motion vectors from + :func:`giant_python.bandsilo.background.bin_motion`. + mot_inds : ndarray + Per-frame motion-bin indices (used to pick the most common bin). + frames_to_keep : ndarray of bool + Frames contributing to the mean. + ref_d, ref_r, ref_c : ndarray + Per-superpixel reference depth/row/column indices. + num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row : int + Output grid geometry. + num_channels : int + Number of acquisition channels (>= 2 fills the second channel). + low_res_data2_norm : ndarray, optional + Count-normalized channel-2 traces (used when ``num_channels >= 2``). + + Returns + ------- + ndarray of float32 + ``mean_im`` with shape + ``(num_channels, num_fast_zs, dmd_pixels_per_column, + dmd_pixels_per_row)``. + """ + mean_im = np.full( + (num_channels, num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row), + np.nan, + dtype=np.float32, + ) + most_common_mot = np.argmax(np.bincount(mot_inds)) + sel_r = ref_r + int(unique_motion[most_common_mot, 0]) + sel_c = ref_c + int(unique_motion[most_common_mot, 1]) + mean_im[0, ref_d, sel_r, sel_c] = np.nanmean( + low_res_data_norm[:, frames_to_keep], axis=1 + ) + if num_channels >= 2: + mean_im[1, ref_d, sel_r, sel_c] = np.nanmean( + low_res_data2_norm[:, frames_to_keep], axis=1 + ) + return mean_im + + +def accumulate_activity_image( + rho: np.ndarray, + sel_pix_idxs: np.ndarray, + num_fast_zs: int, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, + batch_size: int = 1000, +) -> np.ndarray: + """Accumulate squared spatio-temporal local maxima of ``rho``. + + ``rho`` (selected pixels x frames) is scattered into a padded spatial grid + one temporal batch at a time. Within each batch, a voxel is a local maximum + if it strictly exceeds its two temporal neighbors and its four in-plane + neighbors and is not adjacent (after a 3x3 dilation) to any NaN voxel. The + squared values of local maxima are summed into ``act_im``. + + Parameters + ---------- + rho : ndarray of shape (n_sel, n_frames) + Smoothed rho response (NaN allowed). + sel_pix_idxs : ndarray of int + Flattened selected-pixel indices into the spatial grid. + num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row : int + Spatial grid geometry. + batch_size : int + Number of frames processed per batch (capped at ``n_frames``). + + Returns + ------- + ndarray of float32 + ``act_im`` with shape + ``(num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row)``. + """ + n_frames = rho.shape[1] + batch_size = min(batch_size, n_frames) + num_batches = int(np.ceil(n_frames / batch_size)) + + spatial_coords = np.unravel_index( + sel_pix_idxs, (num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row) + ) + depth_indices = spatial_coords[0][None, :].astype(np.intp, copy=False) + row_indices = spatial_coords[1][None, :].astype(np.intp, copy=False) + col_indices = spatial_coords[2][None, :].astype(np.intp, copy=False) + + dilation_struct = np.ones((3, 3), dtype=np.uint8) + act_im = np.zeros( + (num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row), + dtype=np.float32, + ) + temporal_pad = 1 + + # Crop to a padded ROI containing all selected pixels; all-NaN regions + # outside it can never produce a local maximum. + row_min = int(np.min(row_indices)) + row_max = int(np.max(row_indices)) + col_min = int(np.min(col_indices)) + col_max = int(np.max(col_indices)) + row_start = max(0, row_min - 1) + row_stop = min(dmd_pixels_per_column, row_max + 2) + col_start = max(0, col_min - 1) + col_stop = min(dmd_pixels_per_row, col_max + 2) + roi_h = row_stop - row_start + roi_w = col_stop - col_start + + row_indices_roi = (row_indices - row_start).astype(np.intp, copy=False) + col_indices_roi = (col_indices - col_start).astype(np.intp, copy=False) + + prealloc_size = int(min(batch_size + 2 * temporal_pad, n_frames)) + batch_rho = np.empty( + (prealloc_size, num_fast_zs, roi_h, roi_w), dtype=np.float32 + ) + nan_mask = np.empty_like(batch_rho, dtype=bool) + interior_shape = ( + max(prealloc_size - 2, 0), + num_fast_zs, + max(roi_h - 2, 0), + max(roi_w - 2, 0), + ) + local_maxima_core = np.empty(interior_shape, dtype=bool) + compare_tmp_core = np.empty(interior_shape, dtype=bool) + batch_rho_pow2_core = np.empty(interior_shape, dtype=np.float32) + time_indices_pre = np.arange(prealloc_size)[:, None] + + for batch_idx in range(num_batches): + batch_start = batch_idx * batch_size + batch_end = min(batch_start + batch_size, n_frames) + padded_start = max(0, batch_start - temporal_pad) + padded_end = min(n_frames, batch_end + temporal_pad) + curr_size = padded_end - padded_start + + br = batch_rho[:curr_size] + nm = nan_mask[:curr_size] + + br.fill(0) + nm.fill(True) + time_indices = time_indices_pre[:curr_size] + batch_vals = rho[:, padded_start:padded_end].T + br[ + time_indices, depth_indices, row_indices_roi, col_indices_roi + ] = batch_vals + nm[ + time_indices, depth_indices, row_indices_roi, col_indices_roi + ] = np.isnan(batch_vals) + + br[nm] = 0 + core_t = curr_size - 2 + if core_t <= 0: + continue + + dilated_nan_mask = fast_dilation(nm, dilation_struct) + + center = br[1:-1, :, 1:-1, 1:-1] + lmc = local_maxima_core[:core_t] + ctmp = compare_tmp_core[:core_t] + b3c = batch_rho_pow2_core[:core_t] + + np.greater(center, br[:-2, :, 1:-1, 1:-1], out=lmc) + np.greater(center, br[2:, :, 1:-1, 1:-1], out=ctmp) + np.logical_and(lmc, ctmp, out=lmc) + np.greater(center, br[1:-1, :, :-2, 1:-1], out=ctmp) + np.logical_and(lmc, ctmp, out=lmc) + np.greater(center, br[1:-1, :, 2:, 1:-1], out=ctmp) + np.logical_and(lmc, ctmp, out=lmc) + np.greater(center, br[1:-1, :, 1:-1, :-2], out=ctmp) + np.logical_and(lmc, ctmp, out=lmc) + np.greater(center, br[1:-1, :, 1:-1, 2:], out=ctmp) + np.logical_and(lmc, ctmp, out=lmc) + np.logical_not(dilated_nan_mask[1:-1, :, 1:-1, 1:-1], out=ctmp) + np.logical_and(lmc, ctmp, out=lmc) + + np.multiply(center, center, out=b3c) + np.multiply(b3c, lmc, out=b3c, casting="unsafe") + r_lo, r_hi = row_start + 1, row_stop - 1 + c_lo, c_hi = col_start + 1, col_stop - 1 + act_im[:, r_lo:r_hi, c_lo:c_hi] += np.sum( + b3c, axis=0, dtype=np.float32 + ) + + return act_im + + +def finalize_activity_image( + act_im: np.ndarray, + sel_pix_idxs: np.ndarray, + nan_ct: np.ndarray, +) -> np.ndarray: + """Mask never-valid pixels and subtract a local NaN-median. + + Pixels whose rho was mostly NaN (``nan_ct > 0.5``) or that were never + selected are set to NaN; the remaining pixels have a local ``1x11x11`` + NaN-median subtracted, and the mask is re-applied. + + Parameters + ---------- + act_im : ndarray + Accumulated activity image (modified in place before subtraction). + sel_pix_idxs : ndarray of int + Flattened selected-pixel indices into ``act_im``'s grid. + nan_ct : ndarray of shape (n_sel,) + Per-selected-pixel rho NaN fraction from + :func:`giant_python.bandsilo.background.mask_high_nan_rho`. + + Returns + ------- + ndarray + The finalized activity image (NaN outside the valid support). + """ + nan_mask = np.full_like(act_im, True, dtype=bool) + valid_sel_pix = np.flatnonzero(nan_ct <= 0.5) + nan_mask[ + np.unravel_index(sel_pix_idxs[valid_sel_pix], nan_mask.shape) + ] = False + act_im[nan_mask] = np.nan + + med_act_im = ndimage.generic_filter(act_im, np.nanmedian, size=(1, 11, 11)) + act_im = act_im - med_act_im + act_im[nan_mask] = np.nan + return act_im diff --git a/tests/test_bandsilo_background.py b/tests/test_bandsilo_background.py new file mode 100644 index 0000000..f514097 --- /dev/null +++ b/tests/test_bandsilo_background.py @@ -0,0 +1,526 @@ +"""Tests for giant_python.bandsilo.background (Phase 4 kernels). + +Covers the motion-binning, PSF/convolution, interpolated-background, +noise-model, and rho kernels with synthetic fixtures. ``build_interp_data`` is +additionally cross-checked against a verbatim copy of the reference in the +project's development notes; here we assert its structural behavior. The rho +path is exercised end-to-end on a small synthetic geometry built from the real +:mod:`giant_python.bandsilo.geometry` helpers, and checked for linearity in the +residual (rho is a linear projection of the residual). +""" + +import unittest + +import numpy as np +import torch + +from giant_python.bandsilo import background as bg +from giant_python.bandsilo import geometry as geo + + +def _small_geometry(seed=0, positions_per_plane=None): + """Build a small consistent DMD geometry for rho/convolution tests. + + Returns a dict of the geometry arrays, PSF tensors, selected pixels, per + plane convolution matrices, and per-motion H matrices. + + ``positions_per_plane`` is a list (one entry per z-plane) of ``(row, col)`` + reference-pixel positions. It defaults to a dense 7x7 block in every plane + so the wide horizontal erosion in compute_rho leaves a non-empty valid + region. + """ + rng = np.random.default_rng(seed) + npc = 15 # dmd_pixels_per_column + npr = 15 # dmd_pixels_per_row + plane_size = npc * npr + + if positions_per_plane is None: + block = [(r, c) for r in range(4, 11) for c in range(4, 11)] + positions_per_plane = [block, block] + num_fast_zs = len(positions_per_plane) + + rows = [] + for z, positions in enumerate(positions_per_plane): + for r, c in positions: + rows.append([z * plane_size + r * npr + c, len(rows) + 1]) + subsample_matrix_inds = np.array(rows, dtype=np.int32) + num_super_pixels = subsample_matrix_inds.shape[0] + + # A small normalized 5x5 PSF (odd so it has a well-defined center). + yy, xx = np.mgrid[-2:3, -2:3] + psf2d = np.exp(-(yy**2 + xx**2) / 2.0).astype(np.float32) + + ref_d, ref_c, ref_r = geo.ref_pixs_to_drc( + subsample_matrix_inds[:, 0], npc, npr + ) + sparse_h_inds, sparse_h_vals = geo.build_sparse_h( + subsample_matrix_inds, psf2d, npc, npr + ) + + umyx = np.array([[0, 0], [1, 0]], dtype=np.float64) + sel_pix_mask, sel_pix_idxs = bg.build_selected_pixel_mask( + umyx, ref_d, ref_r, ref_c, num_fast_zs, npc, npr, psf2d + ) + + psf_tensor, psf_center, psf_tensor_exp, psf_center_exp = bg.expand_psf( + psf2d + ) + + d_mats = [] + d_mats_exp = [] + for z in range(num_fast_zs): + _, sel_2d = bg.selected_pixels_2d_for_plane(sel_pix_idxs, z, npc, npr) + d_mats.append( + bg.build_convolution_matrix(sel_2d, psf_tensor, psf_center) + ) + d_mats_exp.append( + bg.build_convolution_matrix(sel_2d, psf_tensor_exp, psf_center_exp) + ) + + h_mots = bg.build_motion_h_matrices( + sparse_h_inds, sparse_h_vals, umyx, sel_pix_idxs, num_super_pixels, npr + ) + + return dict( + rng=rng, + npc=npc, + npr=npr, + num_fast_zs=num_fast_zs, + num_super_pixels=num_super_pixels, + psf2d=psf2d, + ref_d=ref_d, + ref_r=ref_r, + ref_c=ref_c, + umyx=umyx, + sel_pix_idxs=sel_pix_idxs, + d_mats=d_mats, + d_mats_exp=d_mats_exp, + h_mots=h_mots, + ) + + +class TestMotionBinning(unittest.TestCase): + """bin_motion / select_motion_bins / bin_motion_yx.""" + + def test_bin_motion_groups_identical_rounded_vectors(self): + """Frames with equal rounded motion share a bin index.""" + mr = np.array([0.1, 0.2, 5.0]) + mc = np.array([0.0, 0.0, 0.0]) + mz = np.array([0.0, 0.0, 0.0]) + unique_motion, mot_inds = bg.bin_motion(mr, mc, mz) + self.assertEqual(unique_motion.shape[1], 3) + self.assertEqual(mot_inds[0], mot_inds[1]) + self.assertNotEqual(mot_inds[0], mot_inds[2]) + self.assertEqual(mot_inds.ndim, 1) + + def test_select_motion_bins_filters_z_and_count(self): + """Bins far in z or with too few frames are dropped.""" + # bin 0: z=0, many frames; bin 1: z=0, few frames; bin 2: z=10, many + unique_motion = np.array( + [[0, 0, 0], [1, 0, 0], [2, 0, 10]], dtype=float + ) + mot_inds = np.array([0] * 5 + [1] * 2 + [2] * 5) + motion_z = np.zeros(12) + keep, frames = bg.select_motion_bins( + unique_motion, mot_inds, motion_z, z_thresh=1.5, min_frames=3 + ) + self.assertEqual(list(keep), [0]) + self.assertEqual(int(frames.sum()), 5) + + def test_bin_motion_yx_marks_dropped_frames(self): + """Frames outside frames_to_keep get index -1.""" + mr = np.array([0.0, 0.0, 1.0, 1.0]) + mc = np.array([0.0, 0.0, 0.0, 0.0]) + frames_to_keep = np.array([True, True, True, False]) + umyx, mot_inds_yx = bg.bin_motion_yx(mr, mc, frames_to_keep) + self.assertEqual(mot_inds_yx[3], -1) + self.assertEqual(mot_inds_yx[0], mot_inds_yx[1]) + self.assertEqual(umyx.shape[1], 2) + + +class TestSelectedPixels(unittest.TestCase): + """build_selected_pixel_mask / pixel_coords / selected_pixels_2d.""" + + def test_mask_and_coords_round_trip(self): + """Flat selected indices decode back to their z/row/col.""" + ref_d = np.array([0]) + ref_r = np.array([5]) + ref_c = np.array([5]) + umyx = np.array([[0, 0]], dtype=float) + psf2d = np.ones((3, 3), dtype=np.float32) + mask, idxs = bg.build_selected_pixel_mask( + umyx, ref_d, ref_r, ref_c, 1, 12, 12, psf2d + ) + self.assertEqual(int(mask.sum()), idxs.size) + coords = bg.pixel_coords_from_idxs(idxs, 12, 12) + # every decoded coordinate should be marked in the mask + for z, r, c in coords: + self.assertTrue(mask[z, r, c]) + + def test_selected_pixels_2d_for_plane(self): + """Plane selection returns the in-plane row/col for that z only.""" + idxs = np.array([0 * 25 + 2 * 5 + 3, 1 * 25 + 4 * 5 + 1]) + z0, sel0 = bg.selected_pixels_2d_for_plane(idxs, 0, 5, 5) + self.assertEqual(sel0.tolist(), [[2, 3]]) + z1, sel1 = bg.selected_pixels_2d_for_plane(idxs, 1, 5, 5) + self.assertEqual(sel1.tolist(), [[4, 1]]) + + +class TestPsfConvolution(unittest.TestCase): + """expand_psf and build_convolution_matrix.""" + + def test_expand_psf_normalized_and_larger(self): + """Both PSFs sum to 1 and the expanded one is ex_fac larger.""" + yy, xx = np.mgrid[-2:3, -2:3] + psf2d = np.exp(-(yy**2 + xx**2) / 2.0).astype(np.float32) + pt, pc, pte, pce = bg.expand_psf(psf2d, ex_fac=2) + self.assertAlmostEqual(float(pt.sum()), 1.0, places=5) + self.assertAlmostEqual(float(pte.sum()), 1.0, places=5) + self.assertEqual(tuple(pte.shape), (10, 10)) + self.assertEqual(pc, (2, 2)) + self.assertEqual(pce, (5, 5)) + + def test_convolution_matrix_values(self): + """D[t, s] equals the PSF weight at the target-source offset.""" + yy, xx = np.mgrid[-2:3, -2:3] + psf2d = np.exp(-(yy**2 + xx**2) / 2.0).astype(np.float32) + pt, pc, _, _ = bg.expand_psf(psf2d) + sel_2d = np.array([[5, 5], [6, 5], [5, 8]]) + d = bg.build_convolution_matrix(sel_2d, pt, pc) + # self entry == center weight + self.assertAlmostEqual(float(d[0, 0]), float(pt[pc[0], pc[1]]), 5) + # offset (1, 0): target row+1 -> psf[center+1, center] + self.assertAlmostEqual( + float(d[1, 0]), float(pt[pc[0] + 1, pc[1]]), places=5 + ) + # offset (0, 3): rel_col = center + 3 = 5 is outside the 5x5 PSF -> 0 + self.assertEqual(float(d[2, 0]), 0.0) + + def test_convolution_matrix_empty(self): + """An empty plane yields a 0x0 matrix.""" + pt = torch.ones((3, 3)) + d = bg.build_convolution_matrix( + np.zeros((0, 2), dtype=int), pt, (1, 1) + ) + self.assertEqual(tuple(d.shape), (0, 0)) + + +class TestInterpData(unittest.TestCase): + """build_interp_data structural behavior and edge branches.""" + + def test_basic_fill_and_bounds(self): + """Interpolation fills selected pixels within the motion window.""" + ref_r = np.array([4, 6, 8], dtype=np.int32) + ref_c = np.array([5, 5, 5], dtype=np.int32) + data = np.array([[1.0], [3.0], [5.0]], dtype=np.float32) # one frame + sel_2d = np.array([[4, 5], [5, 5], [6, 5], [7, 5]]) + umyx = np.array([[0, 0]], dtype=float) + mot_yx = np.array([0]) + out, rb, cb = bg.build_interp_data( + data, ref_r, ref_c, sel_2d, umyx, mot_yx + ) + # exact rows (4, 6) match data; row 5 interpolates to 2.0 + self.assertAlmostEqual(float(out[0, 0]), 1.0, places=5) + self.assertAlmostEqual(float(out[1, 0]), 2.0, places=5) + self.assertAlmostEqual(float(out[2, 0]), 3.0, places=5) + self.assertEqual(cb, (5, 5)) + + def test_empty_motion_and_missing_column(self): + """Empty bins and columns absent from the selection are skipped.""" + ref_r = np.array([4, 6], dtype=np.int32) + ref_c = np.array([3, 3], dtype=np.int32) # column 3 not selected + data = np.array([[1.0], [2.0]], dtype=np.float32) + sel_2d = np.array([[4, 5], [6, 5]]) # only column 5 selected + umyx = np.array([[0, 0], [1, 0]], dtype=float) + mot_yx = np.array([0]) # bin 1 has no frames + out, _, _ = bg.build_interp_data( + data, ref_r, ref_c, sel_2d, umyx, mot_yx + ) + # column 3 not in selection -> nothing filled + self.assertTrue(np.all(np.isnan(out))) + + def test_single_point_column(self): + """A single-reference-pixel column is copied, not interpolated.""" + ref_r = np.array([5], dtype=np.int32) + ref_c = np.array([5], dtype=np.int32) + data = np.array([[7.0]], dtype=np.float32) + sel_2d = np.array([[5, 5]]) + umyx = np.array([[0, 0]], dtype=float) + mot_yx = np.array([0]) + out, _, _ = bg.build_interp_data( + data, ref_r, ref_c, sel_2d, umyx, mot_yx + ) + self.assertAlmostEqual(float(out[0, 0]), 7.0, places=5) + + def test_all_exact_column_skips_interpolation(self): + """A multi-point column whose targets all match exactly interpolates + nothing (the interp branch is empty).""" + ref_r = np.array([4, 6], dtype=np.int32) + ref_c = np.array([5, 5], dtype=np.int32) + data = np.array([[1.0], [3.0]], dtype=np.float32) + # selected rows are exactly the reference rows -> all exact matches + sel_2d = np.array([[4, 5], [6, 5]]) + umyx = np.array([[0, 0]], dtype=float) + mot_yx = np.array([0]) + out, _, _ = bg.build_interp_data( + data, ref_r, ref_c, sel_2d, umyx, mot_yx + ) + self.assertAlmostEqual(float(out[0, 0]), 1.0, places=5) + self.assertAlmostEqual(float(out[1, 0]), 3.0, places=5) + + def test_out_of_window_motion(self): + """A motion shift pushing all refs outside the window fills nothing.""" + ref_r = np.array([4, 6], dtype=np.int32) + ref_c = np.array([5, 5], dtype=np.int32) + data = np.array([[1.0], [2.0]], dtype=np.float32) + sel_2d = np.array([[4, 5], [6, 5]]) + # single bin, but with a huge column shift beyond width via tiny W + umyx = np.array([[0, 0]], dtype=float) + mot_yx = np.array([0]) + out, _, _ = bg.build_interp_data( + data, ref_r, ref_c, sel_2d, umyx, mot_yx, height=1, width=1 + ) + self.assertTrue(np.all(np.isnan(out))) + + +class TestRollingBaseline(unittest.TestCase): + """baseline_window_frames + compute_rolling_baseline.""" + + def test_window_frames(self): + """Window length is int(hz * seconds).""" + self.assertEqual(bg.baseline_window_frames(80.0, 4.0), 320) + + def test_no_nan_matches_uniform_filter(self): + """With no NaNs, the baseline reduces to a plain moving average.""" + import scipy.ndimage as ndimage + + data = np.arange(20, dtype=np.float32).reshape(2, 10) + expected = ndimage.uniform_filter1d( + data.copy(), size=3, axis=1, mode="nearest" + ) + out = bg.compute_rolling_baseline(data.copy(), 3) + np.testing.assert_allclose(out, expected, rtol=1e-5) + + def test_all_nan_row_stays_nan(self): + """A fully-NaN row yields a fully-NaN baseline.""" + data = np.full((1, 6), np.nan, dtype=np.float32) + out = bg.compute_rolling_baseline(data, 3) + self.assertTrue(np.all(np.isnan(out))) + + +class TestBackgroundAssembly(unittest.TestCase): + """assemble_background maps selected-pixel background to superpixels.""" + + def test_maps_reference_pixels(self): + """Each superpixel receives its shifted reference-pixel background.""" + npc = npr = 10 + plane = npc * npr + ref_d = np.array([0, 0]) + ref_r = np.array([2, 4]) + ref_c = np.array([3, 5]) + # sel pixels are exactly the (unshifted) reference pixels + sel_pix_idxs = np.sort( + np.array([0 * plane + 2 * npr + 3, 0 * plane + 4 * npr + 5]) + ) + interp_bg = np.array([[1.0, 1.0], [2.0, 2.0]], dtype=np.float32) + umyx = np.array([[0, 0]], dtype=float) + mot_yx = np.array([0, 0]) + out = bg.assemble_background( + interp_bg, + umyx, + mot_yx, + sel_pix_idxs, + ref_d, + ref_r, + ref_c, + n_super_pixels=2, + n_frames=2, + dmd_pixels_per_column=npc, + dmd_pixels_per_row=npr, + ) + self.assertFalse(np.any(np.isnan(out))) + self.assertEqual(out.shape, (2, 2)) + + +class TestNoiseModel(unittest.TestCase): + """fit_noise_variance_model + compute_residual.""" + + def test_data_std_positive_and_residual_zscored(self): + """data_std is positive; residual is (data-bg)/std.""" + rng = np.random.default_rng(1) + # Per-pixel mean/count spread so var_pred varies (else the bright + # selection is empty and the fit degenerates). + means = np.linspace(1.0, 10.0, 30).astype(np.float32) + background = np.tile(means[:, None], (1, 200)) + low = background + rng.standard_normal((30, 200)).astype(np.float32) + v_im = np.tile( + (0.05 + 0.01 * np.arange(30)).astype(np.float32)[:, None], + (1, 200), + ) + data_std, v_k, v_b = bg.fit_noise_variance_model( + low, background, v_im, vif=1.38 + ) + self.assertTrue(np.all(data_std > 0)) + self.assertTrue(np.isfinite(v_b)) + residual = bg.compute_residual(low, background, data_std) + np.testing.assert_allclose( + residual, (low - background) / data_std, rtol=1e-6 + ) + + +class TestRho(unittest.TestCase): + """build_motion_h_matrices + compute_rho + smoothing.""" + + def test_h_matrices_shape(self): + """One sparse H per motion bin, sized (superpixels, sel pixels).""" + g = _small_geometry() + self.assertEqual(len(g["h_mots"]), len(g["umyx"])) + h0 = g["h_mots"][0] + self.assertEqual( + tuple(h0.size()), + (g["num_super_pixels"], g["sel_pix_idxs"].shape[0]), + ) + + def test_compute_rho_shape_and_linearity(self): + """rho has the selected-pixel shape and is linear in the residual.""" + g = _small_geometry() + n_frames = 12 + n_sp = g["num_super_pixels"] + residual = ( + g["rng"].standard_normal((n_sp, n_frames)).astype(np.float32) + ) + # assign frames to the two motion bins + mot_yx = np.array([0, 1] * (n_frames // 2), dtype=np.int32) + + kwargs = dict( + unique_motion_to_keep_yx=g["umyx"], + h_mots=g["h_mots"], + d_mats=g["d_mats"], + d_mats_expanded=g["d_mats_exp"], + sel_pix_idxs=g["sel_pix_idxs"], + ref_d=g["ref_d"], + ref_r=g["ref_r"], + ref_c=g["ref_c"], + num_fast_zs=g["num_fast_zs"], + dmd_pixels_per_column=g["npc"], + dmd_pixels_per_row=g["npr"], + psf2d=g["psf2d"], + ) + rho = bg.compute_rho(residual, mot_yx, **kwargs) + self.assertEqual(rho.shape, (g["sel_pix_idxs"].shape[0], n_frames)) + self.assertTrue(np.any(np.isfinite(rho))) + + rho2 = bg.compute_rho(2.0 * residual, mot_yx, **kwargs) + finite = np.isfinite(rho) & np.isfinite(rho2) + np.testing.assert_allclose( + rho2[finite], 2.0 * rho[finite], rtol=1e-4, atol=1e-5 + ) + + def test_compute_rho_handles_empty_motion_and_empty_plane(self): + """A frameless motion bin and an empty z-plane are skipped.""" + g = _small_geometry() + n_frames = 6 + residual = ( + g["rng"] + .standard_normal((g["num_super_pixels"], n_frames)) + .astype(np.float32) + ) + # All frames in bin 0; bin 1 gets none -> motion_frames empty branch + mot_yx = np.zeros(n_frames, dtype=np.int32) + # Add a third, always-empty z-plane -> new_ncols == 0 branch + rho = bg.compute_rho( + residual, + mot_yx, + unique_motion_to_keep_yx=g["umyx"], + h_mots=g["h_mots"], + d_mats=g["d_mats"] + [torch.zeros((0, 0))], + d_mats_expanded=g["d_mats_exp"] + [torch.zeros((0, 0))], + sel_pix_idxs=g["sel_pix_idxs"], + ref_d=g["ref_d"], + ref_r=g["ref_r"], + ref_c=g["ref_c"], + num_fast_zs=g["num_fast_zs"] + 1, + dmd_pixels_per_column=g["npc"], + dmd_pixels_per_row=g["npr"], + psf2d=g["psf2d"], + ) + self.assertEqual(rho.shape[1], n_frames) + + def test_compute_rho_skips_plane_with_no_valid_columns(self): + """A plane whose eroded valid region is empty is skipped (its rho + rows stay NaN) while the dense plane is still filled.""" + block = [(r, c) for r in range(4, 11) for c in range(4, 11)] + # plane 1 has a single reference pixel -> erosion wipes its valid + # region even though it has selected pixels and H columns. + g = _small_geometry(positions_per_plane=[block, [(7, 7)]]) + n_frames = 6 + residual = ( + g["rng"] + .standard_normal((g["num_super_pixels"], n_frames)) + .astype(np.float32) + ) + mot_yx = np.array([0, 1] * (n_frames // 2), dtype=np.int32) + rho = bg.compute_rho( + residual, + mot_yx, + unique_motion_to_keep_yx=g["umyx"], + h_mots=g["h_mots"], + d_mats=g["d_mats"], + d_mats_expanded=g["d_mats_exp"], + sel_pix_idxs=g["sel_pix_idxs"], + ref_d=g["ref_d"], + ref_r=g["ref_r"], + ref_c=g["ref_c"], + num_fast_zs=g["num_fast_zs"], + dmd_pixels_per_column=g["npc"], + dmd_pixels_per_row=g["npr"], + psf2d=g["psf2d"], + ) + plane_size = g["npc"] * g["npr"] + z_of_sel = g["sel_pix_idxs"] // plane_size + # plane 0 produced some finite rho; plane 1 (single pixel) did not + self.assertTrue(np.any(np.isfinite(rho[z_of_sel == 0]))) + self.assertTrue(np.all(np.isnan(rho[z_of_sel == 1]))) + + def test_mask_high_nan_rho(self): + """Rows that are mostly NaN are fully NaN'd; fraction is returned.""" + rho = np.array( + [[1.0, 2.0, 3.0, 4.0], [np.nan, np.nan, np.nan, 1.0]], + dtype=np.float32, + ) + nan_ct = bg.mask_high_nan_rho(rho, thresh=0.5) + self.assertAlmostEqual(nan_ct[0], 0.0) + self.assertAlmostEqual(nan_ct[1], 0.75) + self.assertTrue(np.all(np.isnan(rho[1]))) + self.assertFalse(np.any(np.isnan(rho[0]))) + + def test_decay_kernel_normalized(self): + """The decay kernel sums to 1 and rises to its last sample.""" + k = bg.decay_kernel_1d(0.15, 80.0) + self.assertAlmostEqual(float(k.sum()), 1.0, places=6) + self.assertEqual(np.argmax(k), len(k) - 1) + + def test_smooth_rho_preserves_constant(self): + """A finite constant row is preserved; an all-NaN row stays NaN.""" + rho = np.vstack( + [ + np.full((1, 50), 2.0, dtype=np.float32), + np.full((1, 50), np.nan, dtype=np.float32), + ] + ) + k = bg.decay_kernel_1d(0.05, 80.0) + out = bg.smooth_rho(rho, k) + # interior samples of the constant row remain ~2.0 + np.testing.assert_allclose(out[0, 10:40], 2.0, rtol=1e-4) + self.assertTrue(np.all(np.isnan(out[1]))) + + def test_smooth_rho_all_nan_chunk(self): + """A chunk with no finite data is passed through unchanged.""" + rho = np.full((3, 20), np.nan, dtype=np.float32) + k = bg.decay_kernel_1d(0.05, 80.0) + out = bg.smooth_rho(rho, k) + self.assertTrue(np.all(np.isnan(out))) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_bandsilo_summary_images.py b/tests/test_bandsilo_summary_images.py new file mode 100644 index 0000000..478849b --- /dev/null +++ b/tests/test_bandsilo_summary_images.py @@ -0,0 +1,164 @@ +"""Tests for giant_python.bandsilo.summary_images (Phase 4). + +Exercises the mean image, the spatio-temporal local-maxima activity-image +accumulation, and the median-subtraction finalize on small synthetic inputs. +""" + +import unittest + +import numpy as np + +from giant_python.bandsilo import summary_images as si + + +class TestMeanImage(unittest.TestCase): + """compute_mean_image single- and dual-channel.""" + + def test_single_channel(self): + """Channel-1 mean lands at the shifted reference pixels.""" + ref_d = np.array([0, 0]) + ref_r = np.array([1, 2]) + ref_c = np.array([1, 3]) + low = np.array([[1.0, 3.0], [2.0, 4.0]], dtype=np.float32) + unique_motion = np.array([[0, 0, 0]], dtype=float) + mot_inds = np.array([0, 0]) + frames = np.array([True, True]) + mean_im = si.compute_mean_image( + low, + unique_motion, + mot_inds, + frames, + ref_d, + ref_r, + ref_c, + num_fast_zs=1, + dmd_pixels_per_column=5, + dmd_pixels_per_row=5, + num_channels=1, + ) + self.assertEqual(mean_im.shape, (1, 1, 5, 5)) + self.assertAlmostEqual(float(mean_im[0, 0, 1, 1]), 2.0, places=5) + self.assertAlmostEqual(float(mean_im[0, 0, 2, 3]), 3.0, places=5) + + def test_dual_channel_fills_second(self): + """Channel-2 mean is written when num_channels >= 2.""" + ref_d = np.array([0]) + ref_r = np.array([1]) + ref_c = np.array([1]) + low = np.array([[1.0, 3.0]], dtype=np.float32) + low2 = np.array([[10.0, 30.0]], dtype=np.float32) + unique_motion = np.array([[0, 0, 0]], dtype=float) + mot_inds = np.array([0, 0]) + frames = np.array([True, True]) + mean_im = si.compute_mean_image( + low, + unique_motion, + mot_inds, + frames, + ref_d, + ref_r, + ref_c, + num_fast_zs=1, + dmd_pixels_per_column=4, + dmd_pixels_per_row=4, + num_channels=2, + low_res_data2_norm=low2, + ) + self.assertEqual(mean_im.shape, (2, 1, 4, 4)) + self.assertAlmostEqual(float(mean_im[1, 0, 1, 1]), 20.0, places=5) + + +class TestAccumulateActivityImage(unittest.TestCase): + """accumulate_activity_image local-maxima detection and accumulation.""" + + def test_single_spike_is_detected(self): + """One bright voxel exceeding its neighbors accumulates its square.""" + num_fast_zs, npc, npr = 1, 7, 7 + n_pix = npc * npr + sel_pix_idxs = np.arange(n_pix) + n_frames = 5 + rho = np.zeros((n_pix, n_frames), dtype=np.float32) + # spike at center pixel (row 3, col 3), frame 2, value 4 -> 16 + center_flat = 3 * npr + 3 + rho[center_flat, 2] = 4.0 + act_im = si.accumulate_activity_image( + rho, sel_pix_idxs, num_fast_zs, npc, npr + ) + self.assertEqual(act_im.shape, (num_fast_zs, npc, npr)) + self.assertAlmostEqual(float(act_im[0, 3, 3]), 16.0, places=4) + # neighbors did not accumulate + self.assertAlmostEqual(float(act_im[0, 3, 4]), 0.0, places=5) + + def test_nan_neighbor_suppresses_peak(self): + """A NaN adjacent to a candidate peak removes it via dilation.""" + num_fast_zs, npc, npr = 1, 7, 7 + n_pix = npc * npr + sel_pix_idxs = np.arange(n_pix) + n_frames = 5 + rho = np.zeros((n_pix, n_frames), dtype=np.float32) + center_flat = 3 * npr + 3 + rho[center_flat, 2] = 4.0 + rho[3 * npr + 4, 2] = np.nan # NaN right next to the peak + act_im = si.accumulate_activity_image( + rho, sel_pix_idxs, num_fast_zs, npc, npr + ) + self.assertAlmostEqual(float(act_im[0, 3, 3]), 0.0, places=5) + + def test_small_batches_cover_multiple_iterations(self): + """A tiny batch_size forces multiple batches, same output.""" + num_fast_zs, npc, npr = 1, 7, 7 + n_pix = npc * npr + sel_pix_idxs = np.arange(n_pix) + rho = np.zeros((n_pix, 8), dtype=np.float32) + rho[3 * npr + 3, 4] = 3.0 + big = si.accumulate_activity_image( + rho, sel_pix_idxs, num_fast_zs, npc, npr, batch_size=1000 + ) + small = si.accumulate_activity_image( + rho, sel_pix_idxs, num_fast_zs, npc, npr, batch_size=2 + ) + np.testing.assert_allclose(big, small, rtol=1e-5) + + def test_too_few_frames_yields_zero_image(self): + """A batch with <= 2 frames has no temporal interior and accumulates + nothing.""" + num_fast_zs, npc, npr = 1, 7, 7 + n_pix = npc * npr + sel_pix_idxs = np.arange(n_pix) + rho = np.ones((n_pix, 1), dtype=np.float32) # single frame + act_im = si.accumulate_activity_image( + rho, sel_pix_idxs, num_fast_zs, npc, npr + ) + self.assertTrue(np.all(act_im == 0.0)) + + +class TestFinalizeActivityImage(unittest.TestCase): + """finalize_activity_image masking and local median subtraction.""" + + def test_masks_and_subtracts_median(self): + """Never-valid pixels become NaN; valid ones median-subtracted.""" + num_fast_zs, npc, npr = 1, 13, 13 + act_im = np.ones((num_fast_zs, npc, npr), dtype=np.float32) + # all pixels selected and valid (nan_ct = 0) + sel_pix_idxs = np.arange(npc * npr) + nan_ct = np.zeros(sel_pix_idxs.size) + out = si.finalize_activity_image(act_im.copy(), sel_pix_idxs, nan_ct) + # constant image minus its local median -> ~0 everywhere valid + self.assertTrue(np.all(np.isfinite(out))) + np.testing.assert_allclose(out, 0.0, atol=1e-5) + + def test_high_nan_pixels_masked(self): + """Pixels with nan_ct > 0.5 are excluded and set to NaN.""" + num_fast_zs, npc, npr = 1, 13, 13 + act_im = np.ones((num_fast_zs, npc, npr), dtype=np.float32) + sel_pix_idxs = np.array([3 * npr + 3, 3 * npr + 4]) + nan_ct = np.array([0.0, 0.9]) # second pixel dropped + out = si.finalize_activity_image(act_im, sel_pix_idxs, nan_ct) + # the dropped pixel and all unselected pixels are NaN + self.assertTrue(np.isnan(out[0, 3, 4])) + self.assertTrue(np.isnan(out[0, 0, 0])) + self.assertFalse(np.isnan(out[0, 3, 3])) + + +if __name__ == "__main__": + unittest.main() From 7a789938cb8802fd19ca74f711acb0bbe47f0ca2 Mon Sep 17 00:00:00 2001 From: Michael Xie Date: Sun, 26 Jul 2026 09:59:27 -0700 Subject: [PATCH 07/55] fix bug in comment format --- src/giant_python/bandsilo/background.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/giant_python/bandsilo/background.py b/src/giant_python/bandsilo/background.py index 38499c8..b258f7a 100644 --- a/src/giant_python/bandsilo/background.py +++ b/src/giant_python/bandsilo/background.py @@ -878,8 +878,9 @@ def _valid_sel_cols_for_motion( ) valid_pix_mask = ndimage.binary_erosion( valid_pix_mask, + # TODO: check if dimensions here should change structure=np.ones( - (1, psf2d.shape[0], psf2d.shape[1] * 2 - 1), dtype=bool % TODO: check if dimensions here should change + (1, psf2d.shape[0], psf2d.shape[1] * 2 - 1), dtype=bool ), ) valid_pix_idxs = np.flatnonzero(valid_pix_mask) From 6a45d8bafba46439ec536562deccca8227da55c0 Mon Sep 17 00:00:00 2001 From: Michael Xie Date: Sun, 26 Jul 2026 22:50:29 -0700 Subject: [PATCH 08/55] implement source localization step of bandsilo --- src/giant_python/bandsilo/peaks.py | 696 +++++++++++++++++++++++++++++ tests/test_bandsilo_peaks.py | 364 +++++++++++++++ 2 files changed, 1060 insertions(+) create mode 100644 src/giant_python/bandsilo/peaks.py create mode 100644 tests/test_bandsilo_peaks.py diff --git a/src/giant_python/bandsilo/peaks.py b/src/giant_python/bandsilo/peaks.py new file mode 100644 index 0000000..e06fb39 --- /dev/null +++ b/src/giant_python/bandsilo/peaks.py @@ -0,0 +1,696 @@ +"""Gaussian peak detection in the activity image. + +Ported from the peak-detection kernels of +``extractSLAP2IntegrationSources.py`` (ref 307-712, originally +``getActImPeaks.m``). :func:`get_act_im_peaks` finds source seeds in the 3-D +(Z, H, W) activity image produced by +:mod:`giant_python.bandsilo.summary_images`; those seeds initialize the Phase-6 +NMF localization. + +Detection fits integrated isotropic 2-D Gaussians (each Gaussian integrated +over unit pixels) to the activity image with a bounded Levenberg-Marquardt +solver (:func:`_lsq_curvefit`, using the analytical Jacobian from +:func:`_gaussian_peaks_integrated_val_jac`). Per plane, an initial set of local +maxima is fit jointly, then residual peaks are added one per connected +component per round until none exceed the threshold, and finally +under-amplitude peaks are removed. An optional exclusion mask (e.g. user soma +ROIs) suppresses detection in masked regions. + +The reference's nested helpers (``_make_bounds``/``_peak_mask``/ +``_buffer_mask`` and the per-component new-peak step) are lifted to +module-level functions here so each is independently testable. +""" + +from __future__ import annotations + +from typing import Tuple + +import numpy as np +import scipy.ndimage as ndimage +from scipy.special import erf + +# Amplitude rescale applied to raw activity-image peak values when seeding the +# Gaussian fit (matches the reference ``AMP_SCALE = 1 / 0.75``). +_AMP_SCALE = 1.0 / 0.75 + +# MAD -> Gaussian-sigma scale factor (median absolute deviation / 0.6745). +_MAD_TO_SIGMA = 0.6741891400433162 + + +def gaussian_peaks_integrated( + theta: np.ndarray, yxdata: np.ndarray +) -> np.ndarray: + """Evaluate integrated isotropic 2-D Gaussians at selected pixels. + + Each Gaussian is integrated over unit pixels on a regular grid (port of + ``gaussianPeaksIntegrated``). Only the ``M`` requested pixels are evaluated + (the full grid is never materialized). + + Parameters + ---------- + theta : ndarray of shape (N, 4) + Per-Gaussian ``[amp, mu_y, mu_x, sigma]``. + yxdata : ndarray of shape (M, 2) + Pixel-center ``[y, x]`` coordinates. + + Returns + ------- + ndarray of shape (M,) + Predicted value at each pixel: ``sum_n A_n * Iy[y, n] * Ix[x, n]``. + """ + x_int = yxdata[:, 1].astype(np.intp) + y_int = yxdata[:, 0].astype(np.intp) + x_min = int(x_int.min()) + x_max = int(x_int.max()) + y_min = int(y_int.min()) + y_max = int(y_int.max()) + x_idx = x_int - x_min + y_idx = y_int - y_min + + amp = theta[:, 0] + my = theta[:, 1] + mx = theta[:, 2] + s = np.maximum(theta[:, 3], np.finfo(float).eps) + + c = np.sqrt(np.pi / 2) + rt2 = np.sqrt(2.0) + + xc = np.arange(x_min, x_max + 1, dtype=float) + xl = (xc - 0.5)[:, np.newaxis] + xr = (xc + 0.5)[:, np.newaxis] + ix = ( + c + * s[np.newaxis, :] + * ( + erf((xr - mx[np.newaxis, :]) / (rt2 * s[np.newaxis, :])) + - erf((xl - mx[np.newaxis, :]) / (rt2 * s[np.newaxis, :])) + ) + ) + + yc = np.arange(y_min, y_max + 1, dtype=float) + yb = (yc - 0.5)[:, np.newaxis] + yt = (yc + 0.5)[:, np.newaxis] + iy = ( + c + * s[np.newaxis, :] + * ( + erf((yt - my[np.newaxis, :]) / (rt2 * s[np.newaxis, :])) + - erf((yb - my[np.newaxis, :]) / (rt2 * s[np.newaxis, :])) + ) + ) + + return (iy[y_idx, :] * ix[x_idx, :]) @ amp + + +def _gaussian_peaks_integrated_val_jac( + theta: np.ndarray, yxdata: np.ndarray +) -> Tuple[np.ndarray, np.ndarray]: + """Integrated Gaussians with their analytical Jacobian. + + Shares the integrated-profile intermediates between the forward value and + the Jacobian so the LM solver needs one evaluation per iteration instead of + ``4*N`` finite differences. + + Parameters + ---------- + theta : ndarray of shape (N, 4) + Per-Gaussian ``[amp, mu_y, mu_x, sigma]``. + yxdata : ndarray of shape (M, 2) + Pixel-center ``[y, x]`` coordinates. + + Returns + ------- + val : ndarray of shape (M,) + Forward value at each pixel. + jac : ndarray of shape (M, 4*N) + Column-major-per-Gaussian Jacobian (columns ``0::4`` are ``d/dA``, + ``1::4`` are ``d/dmu_y``, ``2::4`` are ``d/dmu_x``, ``3::4`` are + ``d/dsigma``). + """ + x_int = yxdata[:, 1].astype(np.intp) + y_int = yxdata[:, 0].astype(np.intp) + m = len(x_int) + n = theta.shape[0] + + x_min = int(x_int.min()) + x_max = int(x_int.max()) + y_min = int(y_int.min()) + y_max = int(y_int.max()) + x_idx = x_int - x_min + y_idx = y_int - y_min + + amp = theta[:, 0] + my = theta[:, 1] + mx = theta[:, 2] + s = np.maximum(theta[:, 3], np.finfo(float).eps) + + c = np.sqrt(np.pi / 2) + rt2 = np.sqrt(2.0) + inv_rt2s = 1.0 / (rt2 * s[np.newaxis, :]) + + xc = np.arange(x_min, x_max + 1, dtype=float) + xl = (xc - 0.5)[:, np.newaxis] + xr = (xc + 0.5)[:, np.newaxis] + ux_l = (xl - mx[np.newaxis, :]) * inv_rt2s + ux_r = (xr - mx[np.newaxis, :]) * inv_rt2s + erf_ux_l = erf(ux_l) + erf_ux_r = erf(ux_r) + ix = c * s[np.newaxis, :] * (erf_ux_r - erf_ux_l) + + yc = np.arange(y_min, y_max + 1, dtype=float) + yb = (yc - 0.5)[:, np.newaxis] + yt = (yc + 0.5)[:, np.newaxis] + uy_b = (yb - my[np.newaxis, :]) * inv_rt2s + uy_t = (yt - my[np.newaxis, :]) * inv_rt2s + erf_uy_b = erf(uy_b) + erf_uy_t = erf(uy_t) + iy = c * s[np.newaxis, :] * (erf_uy_t - erf_uy_b) + + iy_m = iy[y_idx, :] + ix_m = ix[x_idx, :] + + dval_da = iy_m * ix_m + val = dval_da @ amp + + exp_uyb2 = np.exp(-(uy_b**2)) + exp_uyt2 = np.exp(-(uy_t**2)) + d_iy_dmy_m = (exp_uyb2 - exp_uyt2)[y_idx, :] + dval_dmy = amp[np.newaxis, :] * d_iy_dmy_m * ix_m + + exp_uxl2 = np.exp(-(ux_l**2)) + exp_uxr2 = np.exp(-(ux_r**2)) + d_ix_dmx_m = (exp_uxl2 - exp_uxr2)[x_idx, :] + dval_dmx = amp[np.newaxis, :] * iy_m * d_ix_dmx_m + + sqrt2 = rt2 + d_iy_ds_m = ( + iy / s[np.newaxis, :] + sqrt2 * (uy_b * exp_uyb2 - uy_t * exp_uyt2) + )[y_idx, :] + d_ix_ds_m = ( + ix / s[np.newaxis, :] + sqrt2 * (ux_l * exp_uxl2 - ux_r * exp_uxr2) + )[x_idx, :] + dval_ds = amp[np.newaxis, :] * (d_iy_ds_m * ix_m + iy_m * d_ix_ds_m) + + jac = np.empty((m, 4 * n), dtype=float) + jac[:, 0::4] = dval_da + jac[:, 1::4] = dval_dmy + jac[:, 2::4] = dval_dmx + jac[:, 3::4] = dval_ds + + return val, jac + + +def _lsq_curvefit( + theta0: np.ndarray, + xdata: np.ndarray, + ydata: np.ndarray, + lb_flat: np.ndarray, + ub_flat: np.ndarray, + max_nfev: int = 5000, +) -> np.ndarray: + """Bounded Levenberg-Marquardt fit of integrated Gaussians. + + Reimplements the subset of MATLAB ``lsqcurvefit`` (trust-region-reflective + defaults) used by ``getActImPeaks.m``, but with the analytical Jacobian so + each iteration is a single value+Jacobian evaluation. + + Parameters + ---------- + theta0 : ndarray of shape (N, 4) + Initial ``[amp, mu_y, mu_x, sigma]`` per Gaussian. + xdata : ndarray of shape (M, 2) + Pixel-center ``[y, x]`` coordinates. + ydata : ndarray of shape (M,) + Observed values (``act_im(sel) - mu_bg``). + lb_flat, ub_flat : ndarray of shape (4*N,) + Row-major-flattened lower/upper bounds. + max_nfev : int + Cap on value+Jacobian evaluations. + + Returns + ------- + ndarray of shape (N, 4) + Optimized parameters. + """ + x = np.clip(theta0.ravel().copy(), lb_flat, ub_flat) + + val, jac = _gaussian_peaks_integrated_val_jac(x.reshape(-1, 4), xdata) + r = val - ydata + cost = np.dot(r, r) + nfev = 1 + + lam = 1e-2 # initial damping (MATLAB InitDamping default) + + for _ in range(400): # MATLAB MaxIter default + if nfev >= max_nfev: + break + + jtj = jac.T @ jac + jtr = jac.T @ r + diag_jtj = np.maximum(np.diag(jtj), 1e-8) + + delta = np.linalg.solve(jtj + lam * np.diag(diag_jtj), -jtr) + x_new = np.clip(x + delta, lb_flat, ub_flat) + + val_new, jac_new = _gaussian_peaks_integrated_val_jac( + x_new.reshape(-1, 4), xdata + ) + r_new = val_new - ydata + cost_new = np.dot(r_new, r_new) + nfev += 1 + + if cost_new < cost: + step_norm = np.max(np.abs(x_new - x)) + rel_cost_drop = (cost - cost_new) / max(cost, 1.0) + + x = x_new + r = r_new + jac = jac_new + cost = cost_new + lam = max(lam * 0.1, 1e-10) + + if step_norm < 1e-6 or rel_cost_drop < 1e-6: + break + else: + lam = min(lam * 10.0, 1e10) + + return x.reshape(-1, 4) + + +def _make_bounds( + plocs: np.ndarray, height: int, width: int +) -> Tuple[np.ndarray, np.ndarray]: + """Return flattened LM bounds for peaks located at ``plocs``. + + Amplitude is unbounded above; means are constrained to +/- 1.5 px of the + seed (clipped to the image); sigma to ``[0.35, 5.0]``. + + Parameters + ---------- + plocs : ndarray of shape (N, 2) + Seed ``[y, x]`` locations. + height, width : int + Image extents (for clipping the mean bounds). + + Returns + ------- + lb_flat, ub_flat : ndarray of shape (4*N,) + Row-major-flattened lower/upper bounds. + """ + n = plocs.shape[0] + lb = np.column_stack( + [ + np.zeros(n), + np.maximum(0, plocs[:, 0] - 1.5), + np.maximum(0, plocs[:, 1] - 1.5), + np.ones(n) * 0.35, + ] + ) + ub = np.column_stack( + [ + np.full(n, np.inf), + np.minimum(height - 1, plocs[:, 0] + 1.5), + np.minimum(width - 1, plocs[:, 1] + 1.5), + np.full(n, 5.0), + ] + ) + return lb.ravel(), ub.ravel() + + +def _peak_mask(tf: np.ndarray, height: int, width: int) -> np.ndarray: + """Return a boolean image marking each Gaussian's rounded center.""" + pim = np.zeros((height, width), dtype=bool) + if tf.shape[0] > 0: + iy = np.clip(np.round(tf[:, 1]).astype(int), 0, height - 1) + ix = np.clip(np.round(tf[:, 2]).astype(int), 0, width - 1) + pim[iy, ix] = True + return pim + + +def _buffer_mask(pim: np.ndarray, buffer_size: int) -> np.ndarray: + """Dilate the peak mask by a ``buffer_size`` square (identity if <= 0).""" + if buffer_size > 0: + return ndimage.binary_dilation( + pim, structure=np.ones((buffer_size, buffer_size)) + ) + return pim + + +def _process_cc_new_peak( + new_peak, + thetaf: np.ndarray, + p_locs: np.ndarray, + labeled_full: np.ndarray, + act_im_2d: np.ndarray, + mu_bg: float, + reject_mask: np.ndarray, + height: int, + width: int, +) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """Add and fit one new residual peak within its connected component. + + Appends the candidate to the parameter set, refits every Gaussian whose + center lands in the component, and — if the new peak barely moved (its + center stayed near-integer) — rejects it (refitting the others without it + and marking its pixel so it is not re-proposed). + + Parameters + ---------- + new_peak : tuple + ``((pY, pX), cc_label)`` for the candidate. + thetaf : ndarray of shape (N, 4) + Current Gaussian parameters. + p_locs : ndarray of shape (N, 2) + Current seed locations. + labeled_full : ndarray + Connected-component labels of the current support. + act_im_2d : ndarray + The plane's activity image. + mu_bg : float + Background median subtracted from the fit target. + reject_mask : ndarray of bool + Mutated in place: the rejected peak's pixel is set True. + height, width : int + Image extents. + + Returns + ------- + thetaf, p_locs, cc_mask, cc_yx + Updated parameters/locations plus the component mask and its pixel + coordinates (for the caller's per-round fit-image update). + """ + (py_new, px_new), cc_label = new_peak + amp_new = act_im_2d[py_new, px_new] * _AMP_SCALE + + n_before = thetaf.shape[0] + thetaf = np.vstack([thetaf, [amp_new, float(py_new), float(px_new), 0.5]]) + p_locs = np.vstack([p_locs, [float(py_new), float(px_new)]]) + new_idx = n_before + + cc_mask = labeled_full == cc_label + cc_yx = np.column_stack(np.where(cc_mask)).astype(float) + cc_vals = act_im_2d[cc_mask] - mu_bg + + iy = np.clip(np.round(thetaf[:, 1]).astype(int), 0, height - 1) + ix = np.clip(np.round(thetaf[:, 2]).astype(int), 0, width - 1) + in_cc = cc_mask[iy, ix] + + lb_cc, ub_cc = _make_bounds(p_locs[in_cc], height, width) + thetaf[in_cc] = _lsq_curvefit( + thetaf[in_cc], cc_yx, cc_vals, lb_cc, ub_cc, max_nfev=5000 + ) + + mu_y_new = thetaf[new_idx, 1] + mu_x_new = thetaf[new_idx, 2] + if ( + abs(mu_y_new - round(mu_y_new)) < 1e-3 + and abs(mu_x_new - round(mu_x_new)) < 1e-3 + ): + refit_mask = in_cc.copy() + refit_mask[new_idx] = False + if np.any(refit_mask): + lb_rf, ub_rf = _make_bounds(p_locs[refit_mask], height, width) + thetaf[refit_mask] = _lsq_curvefit( + thetaf[refit_mask], cc_yx, cc_vals, lb_rf, ub_rf, max_nfev=5000 + ) + thetaf = np.delete(thetaf, new_idx, axis=0) + p_locs = np.delete(p_locs, new_idx, axis=0) + reject_mask[py_new, px_new] = True + + return thetaf, p_locs, cc_mask, cc_yx + + +def _refine_residual_peaks( + thetaf: np.ndarray, + p_locs: np.ndarray, + act_im_2d: np.ndarray, + exclusion_mask: np.ndarray, + mu_bg: float, + sigma_bg: float, + peak_th: float, + buffer_size: int, + height: int, + width: int, + buffer_mask: np.ndarray, + res_im: np.ndarray, + fit_support: np.ndarray, + act_sel_pix: np.ndarray, + fit_im: np.ndarray, +) -> np.ndarray: + """Iteratively add residual peaks, one per connected component per round. + + Each round finds the strongest normalized-residual pixel in every component + and, for those above ``peak_th``, adds and fits a new Gaussian (see + :func:`_process_cc_new_peak`). The fit image, residual, support, and buffer + are rebuilt once per round. Terminates when no component yields a new peak. + + Parameters + ---------- + thetaf, p_locs : ndarray + Current Gaussian parameters and seed locations. + act_im_2d, exclusion_mask : ndarray + The plane's activity image and its exclusion mask. + mu_bg, sigma_bg : float + Background median and MAD-scaled sigma. + peak_th : float + Residual threshold (in sigma) for accepting a new peak. + buffer_size : int + Peak-buffer dilation size. + height, width : int + Image extents. + buffer_mask, res_im, fit_support, act_sel_pix, fit_im : ndarray + Initial per-round state from the first fit. + + Returns + ------- + ndarray of shape (N, 4) + The refined Gaussian parameters. + """ + reject_mask = np.zeros((height, width), dtype=bool) + labeled_full, _ = ndimage.label(act_sel_pix) + + while True: + e = res_im.copy() + e[buffer_mask | exclusion_mask | reject_mask | ~fit_support] = -np.inf + e[np.isnan(e)] = -np.inf + + n_labels = int(labeled_full.max()) + if n_labels == 0: + break + + label_ids = list(range(1, n_labels + 1)) + max_vals = ndimage.maximum(e, labeled_full, label_ids) + max_pos = ndimage.maximum_position(e, labeled_full, label_ids) + + new_peaks = [ + (max_pos[i], label_ids[i]) + for i in range(n_labels) + if max_vals[i] > peak_th + ] + if not new_peaks: + break + + modified_ccs = [] + for new_peak in new_peaks: + thetaf, p_locs, cc_mask, cc_yx = _process_cc_new_peak( + new_peak, + thetaf, + p_locs, + labeled_full, + act_im_2d, + mu_bg, + reject_mask, + height, + width, + ) + modified_ccs.append((cc_mask, cc_yx)) + + p_im = _peak_mask(thetaf, height, width) + buffer_mask = _buffer_mask(p_im, buffer_size) + for cc_mask, cc_yx in modified_ccs: + fit_im[cc_mask] = gaussian_peaks_integrated(thetaf, cc_yx) + res_im = (act_im_2d - fit_im - mu_bg) / sigma_bg + fit_support = fit_im > 1e-3 + + act_sel_pix = ndimage.binary_dilation(p_im, structure=np.ones((9, 9))) + act_sel_pix &= ~np.isnan(act_im_2d) + labeled_full, _ = ndimage.label(act_sel_pix) + + return thetaf + + +def detect_peaks_2d( + act_im_2d: np.ndarray, + exclusion_mask: np.ndarray, + mu_bg: float, + sigma_bg: float, + peak_thresh: float, + peak_th: float, + buffer_size: int = 0, +) -> np.ndarray: + """Detect Gaussian peaks in one activity-image plane. + + Finds initial local maxima (above ``peak_thresh``), fits them jointly, then + iteratively adds residual peaks (:func:`_refine_residual_peaks`), and + finally drops peaks whose amplitude falls below a sigma-adjusted threshold. + + Parameters + ---------- + act_im_2d : ndarray of shape (H, W) + One activity-image plane (may contain NaNs). + exclusion_mask : ndarray of bool, shape (H, W) + Pixels to exclude from detection. + mu_bg, sigma_bg : float + Global background median and MAD-scaled sigma. + peak_thresh : float + Absolute detection threshold ``mu_bg + peak_th * sigma_bg``. + peak_th : float + Threshold in sigma for residual peaks. + buffer_size : int + If > 0, detected peaks are buffered by a ``buffer_size`` square when + suppressing nearby residual peaks. + + Returns + ------- + ndarray of shape (N, 4) + ``[amp, mu_y, mu_x, sigma]`` per peak, or ``(0, 4)`` if none. + """ + height, width = act_im_2d.shape + empty = np.zeros((0, 4)) + + explored = act_im_2d.copy() + explored[exclusion_mask | np.isnan(explored)] = -np.inf + + rank8 = ndimage.rank_filter(explored, rank=7, size=3) + rank9 = ndimage.maximum_filter(explored, size=3) + p_tmp = (rank8 > peak_thresh) & (explored == rank9) + + if not np.any(p_tmp): + return empty + + py, px = np.where(p_tmp) + amp = act_im_2d[py, px] * _AMP_SCALE + n_peaks = len(py) + + act_sel_pix = ndimage.binary_dilation(p_tmp, structure=np.ones((9, 9))) + act_sel_pix &= ~np.isnan(act_im_2d) + + thetaf = np.column_stack( + [amp, py.astype(float), px.astype(float), 0.5 * np.ones(n_peaks)] + ) + p_locs = np.column_stack([py.astype(float), px.astype(float)]) + + sel_yx = np.column_stack(np.where(act_sel_pix)).astype(float) + sel_vals = act_im_2d[act_sel_pix] - mu_bg + + lb_f, ub_f = _make_bounds(p_locs, height, width) + thetaf = _lsq_curvefit(thetaf, sel_yx, sel_vals, lb_f, ub_f, max_nfev=5000) + + p_im = _peak_mask(thetaf, height, width) + buffer_mask = _buffer_mask(p_im, buffer_size) + + fit_im = np.zeros((height, width), dtype=float) + fit_im[act_sel_pix] = gaussian_peaks_integrated(thetaf, sel_yx) + res_im = (act_im_2d - fit_im - mu_bg) / sigma_bg + fit_support = fit_im > 1e-3 + + thetaf = _refine_residual_peaks( + thetaf, + p_locs, + act_im_2d, + exclusion_mask, + mu_bg, + sigma_bg, + peak_th, + buffer_size, + height, + width, + buffer_mask, + res_im, + fit_support, + act_sel_pix, + fit_im, + ) + + if thetaf.shape[0] > 0: + s = thetaf[:, 3] + adj_thresh = peak_thresh / ( + np.pi / 2 * s**2 * erf(1 / (np.sqrt(2) * s)) ** 2 + ) + thetaf = thetaf[thetaf[:, 0] >= adj_thresh] + + return thetaf + + +def get_act_im_peaks( + act_im: np.ndarray, + peak_th: float = 3.0, + exclusion_mask: np.ndarray = None, + buffer_size: int = 0, +) -> np.ndarray: + """Find Gaussian source seeds in a 3-D (Z, H, W) activity image. + + Background statistics (median and MAD-scaled sigma) and the detection + threshold are computed once across all planes for uniform sensitivity, then + each plane is detected independently (:func:`detect_peaks_2d`). + + Parameters + ---------- + act_im : ndarray of shape (Z, H, W) + Activity image (may contain NaNs). + peak_th : float + Threshold in MAD-normalized standard deviations. + exclusion_mask : None or ndarray + ``None``, a 2-D ``(H, W)`` mask applied to every plane, or a 3-D + ``(Z, H, W)`` per-plane mask (e.g. user soma ROIs). + buffer_size : int + If > 0, detected peaks are buffered by a ``buffer_size`` square when + suppressing nearby residual peaks. + + Returns + ------- + ndarray of shape (N, 3) + Source seeds ``[z, mu_y, mu_x]``, or ``(0, 3)`` if none detected. + """ + n_z, height, width = act_im.shape + empty = np.zeros((0, 3)) + + if exclusion_mask is None: + excl_planes = [np.zeros((height, width), dtype=bool)] * n_z + elif exclusion_mask.ndim == 2: + excl_planes = [exclusion_mask.astype(bool)] * n_z + else: + excl_planes = [exclusion_mask[z].astype(bool) for z in range(n_z)] + + valid_vals = act_im[~np.isnan(act_im)] + if valid_vals.size == 0: + return empty + + mu_bg = float(np.nanmedian(act_im)) + sigma_bg = ( + float(np.median(np.abs(valid_vals - np.median(valid_vals)))) + / _MAD_TO_SIGMA + ) + if sigma_bg <= 0: + return empty + + peak_thresh = mu_bg + peak_th * sigma_bg + + source_seeds_list = [] + for z in range(n_z): + thetaf_z = detect_peaks_2d( + act_im[z], + excl_planes[z], + mu_bg, + sigma_bg, + peak_thresh, + peak_th, + buffer_size=buffer_size, + ) + if thetaf_z.shape[0] > 0: + z_col = np.full((thetaf_z.shape[0], 1), z, dtype=float) + source_seeds_list.append( + np.column_stack([z_col, thetaf_z[:, 1], thetaf_z[:, 2]]) + ) + + return np.vstack(source_seeds_list) if source_seeds_list else empty diff --git a/tests/test_bandsilo_peaks.py b/tests/test_bandsilo_peaks.py new file mode 100644 index 0000000..a0a287a --- /dev/null +++ b/tests/test_bandsilo_peaks.py @@ -0,0 +1,364 @@ +"""Tests for giant_python.bandsilo.peaks (Phase 5 peak detection). + +Covers the integrated-Gaussian forward/Jacobian kernels, the bounded LM +solver, the per-plane detector, and the 3-D ``get_act_im_peaks`` driver with +synthetic activity images. The full detector pipeline is additionally +cross-checked against a verbatim copy of the reference in the project's +development notes; here we assert structural/numerical behavior on small cases. +""" + +import unittest + +import numpy as np + +from giant_python.bandsilo import peaks as pk + + +def _grid(height, width): + """Return an ``(H*W, 2)`` array of all ``[y, x]`` pixel centers.""" + yy, xx = np.mgrid[0:height, 0:width] + return np.column_stack( + [yy.ravel().astype(float), xx.ravel().astype(float)] + ) + + +def _single_gaussian_plane(height, width, cy, cx, amp, sigma): + """Render one integrated Gaussian over an ``(H, W)`` plane.""" + yx = _grid(height, width) + theta = np.array([[amp, cy, cx, sigma]]) + return pk.gaussian_peaks_integrated(theta, yx).reshape(height, width) + + +class TestGaussianForward(unittest.TestCase): + """gaussian_peaks_integrated analytic properties.""" + + def test_total_integral(self): + """Integrated Gaussian summed over a wide grid gives amp*2*pi*s^2.""" + yx = _grid(60, 60) + amp, s = 3.0, 2.0 + theta = np.array([[amp, 30.0, 30.0, s]]) + total = pk.gaussian_peaks_integrated(theta, yx).sum() + self.assertAlmostEqual(total, amp * 2 * np.pi * s**2, places=3) + + def test_superposition(self): + """Two Gaussians sum to the sum of their individual renders.""" + yx = _grid(30, 30) + t1 = np.array([[2.0, 10.0, 12.0, 1.5]]) + t2 = np.array([[3.0, 20.0, 18.0, 1.0]]) + both = pk.gaussian_peaks_integrated(np.vstack([t1, t2]), yx) + sep = pk.gaussian_peaks_integrated( + t1, yx + ) + pk.gaussian_peaks_integrated(t2, yx) + np.testing.assert_allclose(both, sep, rtol=1e-10) + + +class TestValJac(unittest.TestCase): + """_gaussian_peaks_integrated_val_jac value + analytical Jacobian.""" + + def test_value_matches_forward(self): + """The val_jac value equals the forward-only evaluation.""" + yx = _grid(20, 20) + theta = np.array([[2.0, 9.3, 11.7, 1.4], [1.5, 14.0, 6.0, 1.1]]) + val, _ = pk._gaussian_peaks_integrated_val_jac(theta, yx) + fwd = pk.gaussian_peaks_integrated(theta, yx) + np.testing.assert_allclose(val, fwd, rtol=1e-12) + + def test_jacobian_matches_finite_difference(self): + """Analytical Jacobian matches a central finite-difference estimate.""" + yx = _grid(16, 16) + theta = np.array([[2.0, 8.3, 7.7, 1.3], [1.2, 11.0, 5.0, 0.9]]) + _, jac = pk._gaussian_peaks_integrated_val_jac(theta, yx) + flat = theta.ravel().copy() + eps = 1e-6 + for k in range(flat.size): + hi = flat.copy() + hi[k] += eps + lo = flat.copy() + lo[k] -= eps + v_hi = pk.gaussian_peaks_integrated(hi.reshape(-1, 4), yx) + v_lo = pk.gaussian_peaks_integrated(lo.reshape(-1, 4), yx) + fd = (v_hi - v_lo) / (2 * eps) + np.testing.assert_allclose(jac[:, k], fd, rtol=1e-4, atol=1e-4) + + +class TestLsqCurvefit(unittest.TestCase): + """_lsq_curvefit recovery and control-flow branches.""" + + def test_recovers_known_gaussian(self): + """Fitting noise-free data recovers the generating parameters.""" + yx = _grid(21, 21) + true = np.array([[5.0, 10.4, 9.6, 1.3]]) + ydata = pk.gaussian_peaks_integrated(true, yx) + theta0 = np.array([[3.0, 10.0, 10.0, 1.0]]) + lb, ub = pk._make_bounds(theta0[:, 1:3], 21, 21) + # widen sigma/amp bounds for a clean recovery + lb[0::4] = 0.0 + ub[0::4] = np.inf + fit = pk._lsq_curvefit(theta0, yx, ydata, lb, ub) + np.testing.assert_allclose(fit[0, 1], 10.4, atol=1e-2) + np.testing.assert_allclose(fit[0, 2], 9.6, atol=1e-2) + np.testing.assert_allclose(fit[0, 3], 1.3, atol=1e-2) + + def test_max_nfev_early_break(self): + """max_nfev=1 returns the (clipped) initial parameters immediately.""" + yx = _grid(11, 11) + ydata = pk.gaussian_peaks_integrated( + np.array([[4.0, 5.0, 5.0, 1.0]]), yx + ) + theta0 = np.array([[3.0, 5.0, 5.0, 1.0]]) + lb, ub = pk._make_bounds(theta0[:, 1:3], 11, 11) + lb[0::4] = 0.0 + ub[0::4] = np.inf + out = pk._lsq_curvefit(theta0, yx, ydata, lb, ub, max_nfev=1) + np.testing.assert_allclose(out, theta0) + + def test_non_improving_step_increases_damping(self): + """A clipped, non-improving step exercises the damping-increase branch. + + Tight mean bounds around a wrong location force steps that cannot + reduce cost, so the solver repeatedly raises the LM damping. + """ + yx = _grid(21, 21) + # data has its peak far from the seed; bounds pin the mean near seed + ydata = pk.gaussian_peaks_integrated( + np.array([[6.0, 15.0, 15.0, 1.2]]), yx + ) + theta0 = np.array([[6.0, 4.0, 4.0, 1.2]]) + lb = np.array([5.9, 3.9, 3.9, 1.1]) + ub = np.array([6.1, 4.1, 4.1, 1.3]) + out = pk._lsq_curvefit(theta0, yx, ydata, lb, ub, max_nfev=50) + # stays within the tight bounds (no crash, no bound violation) + self.assertTrue(np.all(out.ravel() >= lb - 1e-9)) + self.assertTrue(np.all(out.ravel() <= ub + 1e-9)) + + +class TestSmallHelpers(unittest.TestCase): + """_make_bounds / _peak_mask / _buffer_mask.""" + + def test_make_bounds(self): + """Means bound to +/-1.5 px (clipped); sigma to [0.35, 5].""" + plocs = np.array([[0.0, 9.0]]) + lb, ub = pk._make_bounds(plocs, 10, 10) + # y lower clipped to 0; x within range + self.assertAlmostEqual(lb[1], 0.0) + self.assertAlmostEqual(ub[2], 9.0) # min(W-1, 9+1.5) = 9 + self.assertAlmostEqual(lb[3], 0.35) + self.assertAlmostEqual(ub[3], 5.0) + + def test_peak_mask_and_empty(self): + """Peak centers mark the mask; an empty parameter set marks nothing.""" + tf = np.array([[1.0, 3.2, 4.8, 1.0]]) + mask = pk._peak_mask(tf, 8, 8) + self.assertTrue(mask[3, 5]) + self.assertEqual(mask.sum(), 1) + empty = pk._peak_mask(np.zeros((0, 4)), 8, 8) + self.assertFalse(empty.any()) + + def test_buffer_mask(self): + """buffer_size>0 dilates; <=0 returns the mask unchanged.""" + pim = np.zeros((7, 7), dtype=bool) + pim[3, 3] = True + buffered = pk._buffer_mask(pim, 3) + self.assertEqual(buffered.sum(), 9) # 3x3 dilation + same = pk._buffer_mask(pim, 0) + np.testing.assert_array_equal(same, pim) + + +class TestProcessCcNewPeak(unittest.TestCase): + """_process_cc_new_peak accept / reject (with and without other peaks).""" + + def test_rejects_integer_peak_with_others(self): + """A peak that converges to its integer seed, beside an existing one, + is rejected (and the others refit).""" + h = w = 15 + # a blob centered exactly on the integer seed -> fit stays near-integer + act = _single_gaussian_plane(h, w, 5.0, 5.0, 8.0, 1.1) + labeled = np.ones((h, w), dtype=int) + thetaf = np.array([[1.0, 10.0, 10.0, 1.0]]) # existing peak in cc + p_locs = np.array([[10.0, 10.0]]) + reject_mask = np.zeros((h, w), dtype=bool) + new_thetaf, _, _, _ = pk._process_cc_new_peak( + ((5, 5), 1), thetaf, p_locs, labeled, act, 0.0, reject_mask, h, w + ) + self.assertEqual(new_thetaf.shape[0], 1) # new peak removed + self.assertTrue(reject_mask[5, 5]) + + def test_rejects_flat_peak_alone(self): + """A non-moving lone peak is rejected (no other peaks to refit).""" + h = w = 15 + act = np.zeros((h, w)) + labeled = np.zeros((h, w), dtype=int) + labeled[3:12, 3:12] = 1 + thetaf = np.zeros((0, 4)) + p_locs = np.zeros((0, 2)) + reject_mask = np.zeros((h, w), dtype=bool) + new_thetaf, _, _, _ = pk._process_cc_new_peak( + ((5, 5), 1), thetaf, p_locs, labeled, act, 0.0, reject_mask, h, w + ) + self.assertEqual(new_thetaf.shape[0], 0) + self.assertTrue(reject_mask[5, 5]) + + def test_accepts_moving_peak(self): + """A peak that moves off its integer seed is kept.""" + h = w = 15 + act = _single_gaussian_plane(h, w, 5.4, 5.6, 6.0, 1.1) + labeled = np.ones((h, w), dtype=int) + thetaf = np.zeros((0, 4)) + p_locs = np.zeros((0, 2)) + reject_mask = np.zeros((h, w), dtype=bool) + new_thetaf, _, _, _ = pk._process_cc_new_peak( + ((5, 5), 1), thetaf, p_locs, labeled, act, 0.0, reject_mask, h, w + ) + self.assertEqual(new_thetaf.shape[0], 1) + self.assertFalse(reject_mask.any()) + # the kept peak moved toward the true (off-integer) center + self.assertGreater( + abs(new_thetaf[0, 1] - round(new_thetaf[0, 1])), 1e-3 + ) + + +class TestRefineResidualPeaks(unittest.TestCase): + """_refine_residual_peaks early-exit branch.""" + + def test_empty_support_returns_unchanged(self): + """No support (labels max 0) exits immediately, returning thetaf.""" + h = w = 10 + thetaf = np.array([[1.0, 5.0, 5.0, 1.0]]) + p_locs = np.array([[5.0, 5.0]]) + act = np.zeros((h, w)) + empty2d = np.zeros((h, w), dtype=bool) + out = pk._refine_residual_peaks( + thetaf, + p_locs, + act, + empty2d, # exclusion + 0.0, + 1.0, + 3.0, + 0, + h, + w, + empty2d.copy(), # buffer_mask + np.zeros((h, w)), # res_im + np.zeros((h, w), dtype=bool), # fit_support + np.zeros((h, w), dtype=bool), # act_sel_pix -> no labels + np.zeros((h, w)), # fit_im + ) + np.testing.assert_array_equal(out, thetaf) + + +class TestDetectPeaks2d(unittest.TestCase): + """detect_peaks_2d single-plane behavior.""" + + def test_no_peaks_returns_empty(self): + """A plane with nothing above threshold returns no peaks.""" + rng = np.random.default_rng(0) + act = rng.standard_normal((30, 30)) * 0.2 + out = pk.detect_peaks_2d( + act, + np.zeros((30, 30), dtype=bool), + mu_bg=0.0, + sigma_bg=0.2, + peak_thresh=100.0, # unreachable + peak_th=100.0, + ) + self.assertEqual(out.shape, (0, 4)) + + def test_single_blob_detected(self): + """A single blob yields one peak near its true center.""" + act = _single_gaussian_plane(30, 30, 14.3, 16.7, 10.0, 1.3) + out = pk.detect_peaks_2d( + act, + np.zeros((30, 30), dtype=bool), + mu_bg=0.0, + sigma_bg=0.05, + peak_thresh=1.0, + peak_th=3.0, + ) + self.assertGreaterEqual(out.shape[0], 1) + # the strongest peak should sit near the true center + best = out[np.argmax(out[:, 0])] + self.assertAlmostEqual(best[1], 14.3, delta=0.5) + self.assertAlmostEqual(best[2], 16.7, delta=0.5) + + def test_close_pair_adds_residual_peak(self): + """Two blobs too close to both be initial maxima: the second is + recovered by the residual-peak refinement loop.""" + act = _single_gaussian_plane( + 34, 34, 16.4, 15.0, 10.0, 1.2 + ) + _single_gaussian_plane(34, 34, 16.4, 17.5, 9.0, 1.2) + out = pk.detect_peaks_2d( + act, + np.zeros((34, 34), dtype=bool), + mu_bg=0.0, + sigma_bg=0.05, + peak_thresh=1.0, + peak_th=3.0, + ) + # both blobs recovered though only one initial local max exists + self.assertEqual(out.shape[0], 2) + + +class TestGetActImPeaks(unittest.TestCase): + """get_act_im_peaks 3-D driver and its guard branches.""" + + def _act_im(self, seed=0): + """Two-plane activity image with a blob per plane, low noise, NaNs.""" + rng = np.random.default_rng(seed) + act = rng.standard_normal((2, 40, 40)) * 0.3 + act[0] += _single_gaussian_plane(40, 40, 12.4, 18.6, 9.0, 1.3) + act[1] += _single_gaussian_plane(40, 40, 25.0, 15.0, 8.0, 1.2) + act[:, :3, :3] = np.nan + return act + + def test_detects_seeds_per_plane(self): + """Both planes' blobs are found, tagged with their z index.""" + seeds = pk.get_act_im_peaks(self._act_im(), peak_th=4.0, buffer_size=3) + self.assertEqual(seeds.shape[1], 3) + self.assertTrue(np.any(seeds[:, 0] == 0)) + self.assertTrue(np.any(seeds[:, 0] == 1)) + + def test_exclusion_mask_2d(self): + """A 2-D exclusion mask suppresses seeds in the masked region.""" + excl = np.zeros((40, 40), dtype=bool) + excl[8:18, 14:24] = True # covers the plane-0 blob + seeds = pk.get_act_im_peaks( + self._act_im(), peak_th=4.0, exclusion_mask=excl, buffer_size=3 + ) + # no seed inside the excluded box + for z, y, x in seeds: + self.assertFalse(8 <= y < 18 and 14 <= x < 24) + + def test_exclusion_mask_3d(self): + """A 3-D per-plane exclusion mask is applied plane-by-plane.""" + excl = np.zeros((2, 40, 40), dtype=bool) + excl[1, 20:30, 10:20] = True # covers the plane-1 blob only + seeds = pk.get_act_im_peaks( + self._act_im(), peak_th=4.0, exclusion_mask=excl, buffer_size=3 + ) + for z, y, x in seeds: + if int(z) == 1: + self.assertFalse(20 <= y < 30 and 10 <= x < 20) + + def test_all_nan_returns_empty(self): + """An all-NaN activity image yields no seeds.""" + act = np.full((2, 20, 20), np.nan) + out = pk.get_act_im_peaks(act) + self.assertEqual(out.shape, (0, 3)) + + def test_zero_mad_returns_empty(self): + """A constant image (MAD sigma 0) yields no seeds.""" + act = np.ones((2, 20, 20)) + out = pk.get_act_im_peaks(act) + self.assertEqual(out.shape, (0, 3)) + + def test_no_peaks_returns_empty(self): + """Pure low noise with a high threshold yields no seeds.""" + rng = np.random.default_rng(1) + act = rng.standard_normal((2, 30, 30)) * 0.3 + out = pk.get_act_im_peaks(act, peak_th=50.0) + self.assertEqual(out.shape, (0, 3)) + + +if __name__ == "__main__": + unittest.main() From 3180d14dc6bc5d456a18dc24bd86a49eef85be2d Mon Sep 17 00:00:00 2001 From: Michael Xie Date: Mon, 27 Jul 2026 18:50:05 -0700 Subject: [PATCH 09/55] implement lowres nmf --- src/giant_python/bandsilo/nmf.py | 822 ++++++++++++++++++++++++++++++ tests/test_bandsilo_nmf.py | 283 ++++++++++ tests/test_bandsilo_peaks.py | 8 +- tests/test_bandsilo_trial_data.py | 14 + 4 files changed, 1124 insertions(+), 3 deletions(-) create mode 100644 src/giant_python/bandsilo/nmf.py create mode 100644 tests/test_bandsilo_nmf.py diff --git a/src/giant_python/bandsilo/nmf.py b/src/giant_python/bandsilo/nmf.py new file mode 100644 index 0000000..55ffd43 --- /dev/null +++ b/src/giant_python/bandsilo/nmf.py @@ -0,0 +1,822 @@ +"""Superpixel-space NMF source localization. + +Ported from the source-localization block of +``extractSLAP2IntegrationSources.py`` (ref 2519-2908). Given the peak seeds +from :mod:`giant_python.bandsilo.peaks` and the noise-normalized residual / +per-motion ``H`` matrices from :mod:`giant_python.bandsilo.background`, this +module fits Gaussian spatial profiles (``A``) and per-motion temporal weights +(``phi``) that reconstruct the residual in superpixel space. + +Each outer iteration: + +1. runs motion-binned multiplicative NMF (in random motion order) to get a + per-motion superpixel target ``X_mots`` (:func:`multiplicative_nmf`), +2. fits the parametric Gaussian profiles to those targets with Adam + (:func:`fit_gaussian_profiles`), +3. rebuilds ``A`` and least-squares-solves ``phi`` + (:func:`fit_phi_all_motions`), +4. sorts sources by explained variance, and every fourth iteration prunes + sources whose SNR falls below ``1/3``. + +The final per-source low-resolution SNR is returned alongside ``A``, ``phi``, +the fitted ``source_params`` ``[z, y, x, sigma_y, sigma_x, tilt]``, and the +(sorted/pruned) ``source_seeds``. + +This block is intentionally stochastic (random motion order via +``torch.randperm`` in NMF and Adam); the pipeline does not seed the RNG. The +``randperm`` calls are preserved in the same order as the reference so a seeded +run reproduces it exactly. The reference's loss-only ``phi`` recomputation and +``print``/warning diagnostics are dropped (they do not affect the outputs and +consume no RNG). +""" + +from __future__ import annotations + +from typing import List + +import numpy as np +import torch + + +def sel_pix_gaussian_profile( + gaussian_params: torch.Tensor, pixel_coords_tensor: torch.Tensor +) -> torch.Tensor: + """Render normalized bivariate-Gaussian profiles over selected pixels. + + Parameters + ---------- + gaussian_params : torch.Tensor of shape (N, 6) + Per-source ``[z, mu_y, mu_x, sigma_y, sigma_x, tilt]`` (the tilt is + mapped to a correlation coefficient via ``tanh``). + pixel_coords_tensor : torch.Tensor of shape (n_sel, 3) + Selected-pixel ``[z, row, col]`` coordinates. + + Returns + ------- + torch.Tensor of shape (n_sel, N) + Column-normalized profiles; each source is confined to its own z-plane + and to within 3 sigma of its center. + """ + z_planes = gaussian_params[:, 0].unsqueeze(0) + y_means = gaussian_params[:, 1].unsqueeze(0) + x_means = gaussian_params[:, 2].unsqueeze(0) + y_sigmas = gaussian_params[:, 3].unsqueeze(0) + x_sigmas = gaussian_params[:, 4].unsqueeze(0) + corr_coef = torch.tanh(gaussian_params[:, 5].unsqueeze(0)) + + y_centered = pixel_coords_tensor[:, 1].unsqueeze(1) - y_means + x_centered = pixel_coords_tensor[:, 2].unsqueeze(1) - x_means + + z_score_y = y_centered / y_sigmas + z_score_x = x_centered / x_sigmas + + exponent = (-1 / (2 * (1 - corr_coef**2))) * ( + z_score_y**2 - 2 * corr_coef * z_score_x * z_score_y + z_score_x**2 + ) + profile = torch.exp(exponent) + profile = ( + profile + * ( + (torch.sqrt(z_score_y**2 + z_score_x**2) <= 3) + & (pixel_coords_tensor[:, 0].unsqueeze(1) == z_planes) + ).float() + ) + + return profile / torch.sum(profile, dim=0, keepdim=True) + + +def sel_pix_patch_profile( + patch_params: torch.Tensor, pixel_coords_tensor: torch.Tensor +) -> torch.Tensor: + """Render rectangular in-plane support patches over selected pixels. + + Parameters + ---------- + patch_params : torch.Tensor of shape (N, 5) + Per-source ``[z, mu_y, mu_x, y_radius, x_radius]``. + pixel_coords_tensor : torch.Tensor of shape (n_sel, 3) + Selected-pixel ``[z, row, col]`` coordinates. + + Returns + ------- + torch.Tensor of bool, shape (n_sel, N) + True where a pixel is inside a source's patch (same z-plane, within the + radii). + """ + z_planes = patch_params[:, 0].unsqueeze(0) + y_means = patch_params[:, 1].unsqueeze(0) + x_means = patch_params[:, 2].unsqueeze(0) + y_radii = patch_params[:, 3].unsqueeze(0) + x_radii = patch_params[:, 4].unsqueeze(0) + + y_centered = pixel_coords_tensor[:, 1].unsqueeze(1) - y_means + x_centered = pixel_coords_tensor[:, 2].unsqueeze(1) - x_means + + return ( + (torch.abs(y_centered) < y_radii) + & (torch.abs(x_centered) < x_radii) + & (pixel_coords_tensor[:, 0].unsqueeze(1) == z_planes) + ) + + +def init_source_params(source_seeds: np.ndarray) -> torch.Tensor: + """Initialize ``[z, y, x, sigma_y=1, sigma_x=1, tilt=0]`` per source. + + Parameters + ---------- + source_seeds : ndarray of shape (N, 3) + Peak seeds ``[z, mu_y, mu_x]``. + + Returns + ------- + torch.Tensor of shape (N, 6) + The initial source parameters. + """ + n = source_seeds.shape[0] + return torch.cat( + [ + torch.tensor(source_seeds, dtype=torch.float32), + torch.ones(n, 2, dtype=torch.float32), + torch.zeros(n, 1, dtype=torch.float32), + ], + dim=1, + ) + + +def build_a_patches( + source_seeds: np.ndarray, + pixel_coords_tensor: torch.Tensor, + sel_pix_idxs: np.ndarray, + n_pixels: int, + d_xy: float, +) -> torch.Tensor: + """Build the per-source rectangular support mask over the full pixel grid. + + Parameters + ---------- + source_seeds : ndarray of shape (N, 3) + Peak seeds ``[z, mu_y, mu_x]``. + pixel_coords_tensor : torch.Tensor of shape (n_sel, 3) + Selected-pixel coordinates. + sel_pix_idxs : ndarray of int + Flat selected-pixel indices into the ``n_pixels`` grid. + n_pixels : int + Total number of image pixels (``z * rows * cols``). + d_xy : float + Half-width of the square support patch, in pixels. + + Returns + ------- + torch.Tensor of bool, shape (n_pixels, N) + The support mask. + """ + n_sources = source_seeds.shape[0] + a_patches = torch.zeros((n_pixels, n_sources), dtype=torch.bool) + patch_params = torch.cat( + [ + torch.tensor(source_seeds, dtype=torch.float32), + d_xy * torch.ones(n_sources, 2, dtype=torch.float32), + ], + dim=1, + ) + a_patches[sel_pix_idxs, :] = sel_pix_patch_profile( + patch_params, pixel_coords_tensor + ) + return a_patches + + +def build_x_support_mots( + h_mots: List[torch.Tensor], + a_patches: torch.Tensor, + sel_pix_idxs: np.ndarray, + n_motions: int, +) -> List[torch.Tensor]: + """Project each source's support patch into per-motion superpixel space. + + Parameters + ---------- + h_mots : list of torch.Tensor + Per-motion sparse superpixel<-pixel ``H`` matrices. + a_patches : torch.Tensor of bool, shape (n_pixels, N) + Support mask from :func:`build_a_patches`. + sel_pix_idxs : ndarray of int + Flat selected-pixel indices. + n_motions : int + Number of motion bins. + + Returns + ------- + list of torch.Tensor of bool + One ``(n_superpixels, N)`` support mask per motion bin. + """ + return [ + torch.sparse.mm(h_mots[i], a_patches[sel_pix_idxs, :].float()) > 0 + for i in range(n_motions) + ] + + +def project_spatial_profiles( + source_params: torch.Tensor, + pixel_coords_tensor: torch.Tensor, + sel_pix_idxs: np.ndarray, + n_pixels: int, + a_patches: torch.Tensor, +) -> torch.Tensor: + """Build the mass-normalized Gaussian spatial profiles ``A``. + + The Gaussian profile is masked by the support patch and normalized so each + source's column sums to 1. + + Parameters + ---------- + source_params : torch.Tensor of shape (N, 6) + Source parameters. + pixel_coords_tensor : torch.Tensor of shape (n_sel, 3) + Selected-pixel coordinates. + sel_pix_idxs : ndarray of int + Flat selected-pixel indices. + n_pixels : int + Total number of image pixels. + a_patches : torch.Tensor of bool, shape (n_pixels, N) + Support mask. + + Returns + ------- + torch.Tensor of shape (n_pixels, N) + The normalized spatial profiles. + """ + n_sources = source_params.shape[0] + a = torch.zeros((n_pixels, n_sources), dtype=torch.float32) + a[sel_pix_idxs, :] = sel_pix_gaussian_profile( + source_params, pixel_coords_tensor + ) + a[~a_patches] = 0 + mass = torch.sum(a, dim=0, keepdim=True) + mass[mass <= 0] = 1 + a /= mass + return a + + +def _motion_frames(mot_inds_yx: np.ndarray, i: int) -> np.ndarray: + """Return the frame indices belonging to motion bin ``i``.""" + return np.flatnonzero(mot_inds_yx == i) + + +def _solve_phi_motion( + x: torch.Tensor, data_motion: torch.Tensor +) -> torch.Tensor: + """Least-squares temporal weights for one motion's superpixel profiles.""" + xtx = x.T @ x + xtd = x.T @ data_motion + regularized = xtx + 1e-10 * torch.eye(xtx.shape[0]) + return torch.linalg.solve(regularized, xtd).T + + +def fit_phi_all_motions( + a: torch.Tensor, + h_mots: List[torch.Tensor], + sel_pix_idxs: np.ndarray, + data_for_nmf: torch.Tensor, + mot_inds_yx: np.ndarray, + n_motions: int, + phi_low_res: torch.Tensor, +) -> torch.Tensor: + """Least-squares-solve the temporal weights ``phi`` for every motion bin. + + Parameters + ---------- + a : torch.Tensor of shape (n_pixels, N) + Spatial profiles. + h_mots : list of torch.Tensor + Per-motion sparse ``H`` matrices. + sel_pix_idxs : ndarray of int + Flat selected-pixel indices. + data_for_nmf : torch.Tensor of shape (n_superpixels, n_frames) + The residual data being reconstructed. + mot_inds_yx : ndarray of shape (n_frames,) + Per-frame motion-bin index (``-1`` for dropped frames). + n_motions : int + Number of motion bins. + phi_low_res : torch.Tensor of shape (n_frames, N) + Temporal weights, updated in place per motion bin and returned. + + Returns + ------- + torch.Tensor + ``phi_low_res`` with each motion's frames filled. + """ + for i in range(n_motions): + frames = _motion_frames(mot_inds_yx, i) + x = torch.sparse.mm(h_mots[i], a[sel_pix_idxs, :]) + phi_low_res[frames, :] = _solve_phi_motion(x, data_for_nmf[:, frames]) + return phi_low_res + + +def multiplicative_nmf( + a: torch.Tensor, + h_mots: List[torch.Tensor], + x_support_mots: List[torch.Tensor], + sel_pix_idxs: np.ndarray, + data_for_nmf: torch.Tensor, + mot_inds_yx: np.ndarray, + n_motions: int, + sparse_fac: float, + max_iters: int, +) -> List[torch.Tensor]: + """Motion-binned multiplicative NMF, producing per-motion spatial targets. + + Visits motion bins in a random order (``torch.randperm``). For each bin it + projects ``A`` into superpixel space, initializes ``phi`` by projected + least squares, then alternates multiplicative updates of ``phi`` and the + spatial + target ``X`` under the support constraint, periodically sparsifying ``X``. + + Parameters + ---------- + a : torch.Tensor of shape (n_pixels, N) + Current spatial profiles. + h_mots : list of torch.Tensor + Per-motion sparse ``H`` matrices. + x_support_mots : list of torch.Tensor of bool + Per-motion superpixel support masks. + sel_pix_idxs : ndarray of int + Flat selected-pixel indices. + data_for_nmf : torch.Tensor of shape (n_superpixels, n_frames) + Residual data. + mot_inds_yx : ndarray of shape (n_frames,) + Per-frame motion-bin index. + n_motions : int + Number of motion bins. + sparse_fac : float + Sparsification floor subtracted from the max-normalized ``X``. + max_iters : int + Multiplicative-update iterations per motion bin. + + Returns + ------- + list of torch.Tensor + Per-motion fitted superpixel spatial targets ``X_mots``. + """ + x_mots: List[torch.Tensor] = [None] * n_motions + shuffled_indices = torch.randperm(n_motions) + for idx in shuffled_indices: + i = idx.item() + frames = _motion_frames(mot_inds_yx, i) + + x = torch.sparse.mm(h_mots[i], a[sel_pix_idxs, :]) + x[~x_support_mots[i]] = 0 + x = x / torch.norm(x, dim=0, keepdim=True) + + phi = torch.clamp(_solve_phi_motion(x, data_for_nmf[:, frames]), min=0) + + for iter_idx in range(max_iters): + numerator = x.T @ data_for_nmf[:, frames] + denominator = (x.T @ x) @ phi.T + 1e-10 + phi = torch.clamp(phi * (numerator / denominator).T, min=0) + + numerator = data_for_nmf[:, frames] @ phi + denominator = x @ (phi.T @ phi) + 1e-10 + x = torch.clamp(x * (numerator / denominator), min=0) + + x = x * x_support_mots[i].float() + + norms = torch.norm(x, dim=0, keepdim=True) + x = torch.where(norms > 0, x / norms, x) + phi = torch.where(norms > 0, phi * norms, phi) + + if iter_idx % 3 == 0: + x_max = torch.max(x, dim=0, keepdim=True)[0] + x = torch.where(x_max > 0, x / x_max, x) + x = torch.clamp(x, min=sparse_fac) - sparse_fac + norms = torch.norm(x, dim=0, keepdim=True) + x = torch.where(norms > 0, x / norms, x) + + x_mots[i] = x + return x_mots + + +def fit_gaussian_profiles( + source_params: torch.Tensor, + source_seeds: np.ndarray, + x_mots: List[torch.Tensor], + h_mots: List[torch.Tensor], + pixel_coords_tensor: torch.Tensor, + a_patches: torch.Tensor, + sel_pix_idxs: np.ndarray, + d_xy: float, + num_epochs: int, + n_motions: int, + learning_rate: float, + gd_tol: float, +) -> torch.Tensor: + """Adam-fit the Gaussian source parameters to the per-motion NMF targets. + + Optimizes location, scale, and tilt to make the projected Gaussian profiles + match ``X_mots`` (one motion per epoch, in random rounds). Locations are + clamped to +/- ``d_xy`` of the seed and sigmas to ``[0.3, 5]``. Stops early + when consecutive full-round losses differ by less than ``gd_tol``. + + Parameters + ---------- + source_params : torch.Tensor of shape (N, 6) + Current source parameters (z fixed; loc/scale/tilt optimized). + source_seeds : ndarray of shape (N, 3) + Peak seeds (for the location clamp bounds). + x_mots : list of torch.Tensor + Per-motion NMF spatial targets. + h_mots : list of torch.Tensor + Per-motion sparse ``H`` matrices. + pixel_coords_tensor : torch.Tensor of shape (n_sel, 3) + Selected-pixel coordinates. + a_patches : torch.Tensor of bool + Support mask. + sel_pix_idxs : ndarray of int + Flat selected-pixel indices. + d_xy : float + Location clamp half-width. + num_epochs : int + Maximum optimization epochs (one motion each). + n_motions : int + Number of motion bins (epochs per round). + learning_rate : float + Base Adam learning rate (scaled per parameter group). + gd_tol : float + Convergence tolerance on consecutive round losses. + + Returns + ------- + torch.Tensor of shape (N, 6) + The updated source parameters. + """ + optim_loc = source_params[:, 1:3].clone().requires_grad_(True) + optim_scale = source_params[:, 3:5].clone().requires_grad_(True) + optim_tilt = source_params[:, 5].unsqueeze(1).clone().requires_grad_(True) + optimizer = torch.optim.Adam( + [ + {"params": optim_loc, "lr": 10 * learning_rate}, + {"params": optim_scale, "lr": 0.1 * learning_rate}, + {"params": optim_tilt, "lr": 0.1 * learning_rate}, + ] + ) + + z_col = source_params[:, 0].unsqueeze(1) + loc_min = torch.from_numpy(source_seeds[:, 1:3] - d_xy).float() + loc_max = torch.from_numpy(source_seeds[:, 1:3] + d_xy).float() + + losses = [] + shuffled_indices = None + for epoch in range(num_epochs): + optimizer.zero_grad() + + a_step = sel_pix_gaussian_profile( + torch.cat([z_col, optim_loc, optim_scale, optim_tilt], dim=1), + pixel_coords_tensor, + ) + a_step = a_step * a_patches[sel_pix_idxs, :].float() + mass = torch.sum(a_step, dim=0, keepdim=True) + mass[mass <= 0] = 1 + a_step = a_step / mass + + if epoch % n_motions == 0: + shuffled_indices = torch.randperm(n_motions) + i = shuffled_indices[epoch % n_motions].item() + + x_step = torch.sparse.mm(h_mots[i], a_step) + norms = torch.norm(x_step, dim=0, keepdim=True) + x_step = torch.where(norms > 0, x_step / norms, x_step) + loss = torch.sum((x_step - x_mots[i]) ** 2) + + loss.backward() + optimizer.step() + + with torch.no_grad(): + optim_loc.clamp_(min=loc_min, max=loc_max) + optim_scale[:, 0].clamp_(min=0.3, max=5) + optim_scale[:, 1].clamp_(min=0.3, max=5) + + if epoch % n_motions == n_motions - 1: + total_loss = 0 + for i in range(n_motions): + x_step = torch.sparse.mm(h_mots[i], a_step) + norms = torch.norm(x_step, dim=0, keepdim=True) + x_step = torch.where(norms > 0, x_step / norms, x_step) + total_loss += torch.sum((x_step - x_mots[i]) ** 2) + losses.append(total_loss.item()) + if ( + epoch // n_motions > 0 + and abs(losses[-1] - losses[-2]) < gd_tol + ): + break + + return torch.cat( + [z_col, optim_loc.detach(), optim_scale.detach(), optim_tilt.detach()], + dim=1, + ) + + +def variance_sortorder( + phi_low_res: torch.Tensor, n_sources: int +) -> np.ndarray: + """Return the descending-explained-variance source ordering.""" + phi = phi_low_res[:, :n_sources].numpy() + variance = np.nansum((phi - np.nanmean(phi, axis=0)) ** 2, axis=0) + return np.argsort(-variance) + + +def reorder_sources( + order, + source_params: torch.Tensor, + source_seeds: np.ndarray, + a: torch.Tensor, + a_patches: torch.Tensor, + x_support_mots: List[torch.Tensor], + phi_low_res: torch.Tensor, +): + """Apply a source ordering / selection to every per-source array. + + Parameters + ---------- + order : array-like of int + Source indices in the new order (a permutation for sorting, or a subset + for pruning). + source_params, source_seeds, a, a_patches, phi_low_res : array/tensor + Per-source state to reindex along the source axis. + x_support_mots : list of torch.Tensor + Per-motion support masks (reindexed along the source axis). + + Returns + ------- + tuple + The reindexed ``(source_params, source_seeds, a, a_patches, + x_support_mots, phi_low_res)``. + """ + source_params = source_params[order, :] + source_seeds = source_seeds[order, :] + a = a[:, order] + a_patches = a_patches[:, order] + x_support_mots = [support[:, order] for support in x_support_mots] + phi_low_res = phi_low_res[:, order] + return ( + source_params, + source_seeds, + a, + a_patches, + x_support_mots, + phi_low_res, + ) + + +def compute_source_snr( + a: torch.Tensor, + h_mots: List[torch.Tensor], + sel_pix_idxs: np.ndarray, + data_for_nmf: torch.Tensor, + phi_low_res: torch.Tensor, + mot_inds_yx: np.ndarray, + n_motions: int, + n_sources: int, +) -> torch.Tensor: + """Per-source low-resolution SNR (explained variance / residual variance). + + Reconstructs the residual from ``A`` and ``phi``, then for each source sums + the squared explained signal and squared residual over the superpixels the + source contributes to. + + Parameters + ---------- + a : torch.Tensor of shape (n_pixels, N) + Spatial profiles. + h_mots : list of torch.Tensor + Per-motion sparse ``H`` matrices. + sel_pix_idxs : ndarray of int + Flat selected-pixel indices. + data_for_nmf : torch.Tensor of shape (n_superpixels, n_frames) + Residual data. + phi_low_res : torch.Tensor of shape (n_frames, N) + Temporal weights. + mot_inds_yx : ndarray of shape (n_frames,) + Per-frame motion-bin index. + n_motions, n_sources : int + Number of motion bins / sources. + + Returns + ------- + torch.Tensor of shape (N,) + Per-source SNR (``varExp / varResidual``). + """ + residual_recon = torch.full_like(data_for_nmf, float("nan")) + for i in range(n_motions): + frames = _motion_frames(mot_inds_yx, i) + x = torch.sparse.mm(h_mots[i], a[sel_pix_idxs, :]) + residual_recon[:, frames] = ( + data_for_nmf[:, frames] - x @ phi_low_res[frames, :].T + ) + + var_exp = torch.zeros(n_sources, dtype=torch.float32) + var_residual = torch.zeros(n_sources, dtype=torch.float32) + for j in range(n_sources): + for i in range(n_motions): + frames = _motion_frames(mot_inds_yx, i) + x = torch.sparse.mm(h_mots[i], a[sel_pix_idxs, j].unsqueeze(1)) + contributing = (x > 0).nonzero()[:, 0] + var_exp[j] += torch.sum( + torch.sum( + x[contributing] * phi_low_res[frames, j].unsqueeze(-1).T, + dim=0, + ) + ** 2 + ) + var_residual[j] += torch.sum( + torch.sum(residual_recon[contributing][:, frames], dim=0) ** 2 + ) + # TODO: maybe consider making this a peak SNR ratio + return var_exp / var_residual + + +def fit_sources( + source_seeds: np.ndarray, + residual: np.ndarray, + h_mots: List[torch.Tensor], + unique_motion_to_keep_yx: np.ndarray, + mot_inds_yx: np.ndarray, + sel_pix_idxs: np.ndarray, + pixel_coords: np.ndarray, + n_pixels: int, + d_xy: float, + sparse_fac: float, + outer_loop_iters: int = 10, + mult_nmf_max_iters: int = 10, + learning_rate: float = 0.01, + gd_tol: float = 1e-4, +) -> dict: + """Localize sources by superpixel-space NMF + Gaussian-profile fitting. + + Parameters + ---------- + source_seeds : ndarray of shape (N, 3) + Peak seeds ``[z, mu_y, mu_x]`` from :mod:`giant_python.bandsilo.peaks`. + residual : ndarray of shape (n_superpixels, n_frames) + Noise-normalized residual to reconstruct. + h_mots : list of torch.Tensor + Per-motion sparse ``H`` matrices. + unique_motion_to_keep_yx : ndarray of shape (n_motions, 2) + Kept 2-D motion vectors (only its length is used). + mot_inds_yx : ndarray of shape (n_frames,) + Per-frame motion-bin index (``-1`` for dropped frames). + sel_pix_idxs : ndarray of int + Flat selected-pixel indices into the ``n_pixels`` grid. + pixel_coords : ndarray of shape (n_sel, 3) + Selected-pixel ``[z, row, col]`` coordinates. + n_pixels : int + Total number of image pixels. + d_xy : float + Support-patch half-width / location clamp radius. + sparse_fac : float + NMF sparsification floor. + outer_loop_iters : int + Number of outer NMF/fit iterations. + mult_nmf_max_iters : int + Multiplicative-update iterations per motion bin. + learning_rate : float + Base Adam learning rate. + gd_tol : float + Adam convergence tolerance. + + Returns + ------- + dict + ``A`` (n_pixels, N), ``phi_low_res`` (n_frames, N), ``source_params`` + (N, 6), ``source_seeds`` (N, 3), ``source_snr`` (N,), and + ``n_sources``. + """ + n_sources = source_seeds.shape[0] + n_motions = unique_motion_to_keep_yx.shape[0] + n_frames = residual.shape[1] + pixel_coords_tensor = torch.tensor(pixel_coords, dtype=torch.float32) + data_for_nmf = torch.from_numpy(residual.astype(np.float32, copy=False)) + num_epochs = n_motions * 5 + + source_params = init_source_params(source_seeds) + a_patches = build_a_patches( + source_seeds, pixel_coords_tensor, sel_pix_idxs, n_pixels, d_xy + ) + x_support_mots = build_x_support_mots( + h_mots, a_patches, sel_pix_idxs, n_motions + ) + a = project_spatial_profiles( + source_params, pixel_coords_tensor, sel_pix_idxs, n_pixels, a_patches + ) + phi_low_res = torch.full( + (n_frames, n_sources), float("nan"), dtype=torch.float32 + ) + + for outer_loop_iter in range(outer_loop_iters): + x_mots = multiplicative_nmf( + a, + h_mots, + x_support_mots, + sel_pix_idxs, + data_for_nmf, + mot_inds_yx, + n_motions, + sparse_fac, + mult_nmf_max_iters, + ) + source_params = fit_gaussian_profiles( + source_params, + source_seeds, + x_mots, + h_mots, + pixel_coords_tensor, + a_patches, + sel_pix_idxs, + d_xy, + num_epochs, + n_motions, + learning_rate, + gd_tol, + ) + a = project_spatial_profiles( + source_params, + pixel_coords_tensor, + sel_pix_idxs, + n_pixels, + a_patches, + ) + phi_low_res = fit_phi_all_motions( + a, + h_mots, + sel_pix_idxs, + data_for_nmf, + mot_inds_yx, + n_motions, + phi_low_res, + ) + + order = variance_sortorder(phi_low_res, n_sources) + ( + source_params, + source_seeds, + a, + a_patches, + x_support_mots, + phi_low_res, + ) = reorder_sources( + order, + source_params, + source_seeds, + a, + a_patches, + x_support_mots, + phi_low_res, + ) + + if (outer_loop_iter + 1) % 4 == 3: + snr = compute_source_snr( + a, + h_mots, + sel_pix_idxs, + data_for_nmf, + phi_low_res, + mot_inds_yx, + n_motions, + n_sources, + ) + keep = (snr > 1 / 3).nonzero()[:, 0] + n_sources = keep.shape[0] + ( + source_params, + source_seeds, + a, + a_patches, + x_support_mots, + phi_low_res, + ) = reorder_sources( + keep, + source_params, + source_seeds, + a, + a_patches, + x_support_mots, + phi_low_res, + ) + + source_snr = compute_source_snr( + a, + h_mots, + sel_pix_idxs, + data_for_nmf, + phi_low_res, + mot_inds_yx, + n_motions, + n_sources, + ).numpy() + + return { + "A": a, + "phi_low_res": phi_low_res, + "source_params": source_params, + "source_seeds": source_seeds, + "source_snr": source_snr, + "n_sources": n_sources, + } diff --git a/tests/test_bandsilo_nmf.py b/tests/test_bandsilo_nmf.py new file mode 100644 index 0000000..1f13892 --- /dev/null +++ b/tests/test_bandsilo_nmf.py @@ -0,0 +1,283 @@ +"""Tests for giant_python.bandsilo.nmf (Phase 6 source localization). + +Covers the profile/init kernels, the least-squares/SNR helpers, and the full +``fit_sources`` driver (multiplicative NMF + Adam Gaussian fit + variance +sort + SNR pruning) on a small synthetic geometry. The driver is additionally +cross-checked bit-for-bit against a verbatim copy of the reference (under a +shared torch seed) in the project's development notes; here we assert +structural behavior and determinism. +""" + +import unittest + +import numpy as np +import torch + +from giant_python.bandsilo import background as bg +from giant_python.bandsilo import nmf +from tests.test_bandsilo_background import _small_geometry + + +def _tiny_coords(size=5): + """Return (pixel_coords, sel_pix_idxs, n_pixels) for one square plane.""" + rr, cc = np.meshgrid(np.arange(size), np.arange(size), indexing="ij") + coords = np.column_stack( + [np.zeros(size * size), rr.ravel(), cc.ravel()] + ).astype(np.int32) + sel_pix_idxs = np.arange(size * size) + return coords, sel_pix_idxs, size * size + + +class TestProfiles(unittest.TestCase): + """sel_pix_gaussian_profile / sel_pix_patch_profile.""" + + def test_gaussian_profile_normalized_and_confined(self): + """Each Gaussian column sums to 1 and only its z-plane is populated.""" + coords, _, _ = _tiny_coords(5) + pct = torch.tensor(coords, dtype=torch.float32) + params = torch.tensor([[0.0, 2.0, 2.0, 1.0, 1.0, 0.0]]) + prof = nmf.sel_pix_gaussian_profile(params, pct) + self.assertEqual(prof.shape, (25, 1)) + self.assertAlmostEqual(float(prof.sum()), 1.0, places=5) + # peak at the center pixel (row 2, col 2 -> flat index 12) + self.assertEqual(int(torch.argmax(prof[:, 0])), 12) + + def test_gaussian_profile_wrong_plane_empty(self): + """A source on a different z-plane yields an all-nan column.""" + coords, _, _ = _tiny_coords(5) + pct = torch.tensor(coords, dtype=torch.float32) + params = torch.tensor( + [[1.0, 2.0, 2.0, 1.0, 1.0, 0.0]] + ) # z=1, no pixels + prof = nmf.sel_pix_gaussian_profile(params, pct) + # sum is 0 before normalization -> division yields nan + self.assertTrue(torch.all(torch.isnan(prof))) + + def test_patch_profile_membership(self): + """The patch is the strict box within the radii on the source plane.""" + coords, _, _ = _tiny_coords(5) + pct = torch.tensor(coords, dtype=torch.float32) + params = torch.tensor([[0.0, 2.0, 2.0, 2.0, 2.0]]) # radius 2 + patch = nmf.sel_pix_patch_profile(params, pct) + self.assertEqual(patch.shape, (25, 1)) + # |y-2|<2 and |x-2|<2 -> rows/cols {1,2,3} -> 9 pixels + self.assertEqual(int(patch.sum()), 9) + + +class TestInit(unittest.TestCase): + """init_source_params / build_a_patches / project_spatial_profiles.""" + + def test_init_source_params(self): + """Params start as [z, y, x, 1, 1, 0].""" + seeds = np.array([[0.0, 3.0, 4.0], [1.0, 5.0, 6.0]]) + params = nmf.init_source_params(seeds) + self.assertEqual(tuple(params.shape), (2, 6)) + np.testing.assert_array_equal(params[:, 3:5].numpy(), np.ones((2, 2))) + np.testing.assert_array_equal(params[:, 5].numpy(), np.zeros(2)) + + def test_build_a_patches(self): + """Patches are True only within each source's box on the pixel grid.""" + coords, sel, n_pix = _tiny_coords(5) + pct = torch.tensor(coords, dtype=torch.float32) + seeds = np.array([[0.0, 2.0, 2.0]]) + patches = nmf.build_a_patches(seeds, pct, sel, n_pix, d_xy=2) + self.assertEqual(tuple(patches.shape), (25, 1)) + self.assertEqual(int(patches.sum()), 9) + + def test_project_spatial_profiles_normalized_in_patch(self): + """A is normalized to unit column mass and zero outside the patch.""" + coords, sel, n_pix = _tiny_coords(5) + pct = torch.tensor(coords, dtype=torch.float32) + seeds = np.array([[0.0, 2.0, 2.0]]) + params = nmf.init_source_params(seeds) + patches = nmf.build_a_patches(seeds, pct, sel, n_pix, d_xy=2) + a = nmf.project_spatial_profiles(params, pct, sel, n_pix, patches) + self.assertAlmostEqual(float(a.sum()), 1.0, places=5) + # every nonzero pixel lies inside the patch + self.assertTrue(torch.all(patches[a > 0])) + + +class TestVarianceSortAndReorder(unittest.TestCase): + """variance_sortorder / reorder_sources.""" + + def test_variance_sortorder_descending(self): + """Sources are ordered by descending temporal variance.""" + phi = torch.zeros((10, 3)) + phi[:, 0] = torch.linspace(0, 1, 10) # small variance + phi[:, 1] = torch.linspace(0, 10, 10) # large variance + phi[:, 2] = torch.linspace(0, 5, 10) # medium + order = nmf.variance_sortorder(phi, 3) + self.assertEqual(list(order), [1, 2, 0]) + + def test_reorder_sources(self): + """Reordering reindexes every per-source array consistently.""" + order = np.array([2, 0, 1]) + source_params = torch.arange(18).reshape(3, 6).float() + source_seeds = np.arange(9).reshape(3, 3).astype(float) + a = torch.arange(12).reshape(4, 3).float() + a_patches = torch.ones((4, 3), dtype=torch.bool) + x_support = [torch.arange(6).reshape(2, 3)] + phi = torch.arange(15).reshape(5, 3).float() + sp, ss, na, nap, xs, nphi = nmf.reorder_sources( + order, source_params, source_seeds, a, a_patches, x_support, phi + ) + np.testing.assert_array_equal(ss[:, 0], [6.0, 0.0, 3.0]) + np.testing.assert_array_equal(na[0].numpy(), [2.0, 0.0, 1.0]) + np.testing.assert_array_equal(xs[0][0].numpy(), [2, 0, 1]) + + +class TestPhiAndSnr(unittest.TestCase): + """fit_phi_all_motions / build_x_support_mots / compute_source_snr.""" + + def _setup(self, seed=0): + """Build A / data / geometry for a couple of sources.""" + g = _small_geometry(seed=seed) + npc, npr, nz = g["npc"], g["npr"], g["num_fast_zs"] + n_pix = nz * npc * npr + sel = g["sel_pix_idxs"] + pct = torch.tensor( + bg.pixel_coords_from_idxs(sel, npc, npr), dtype=torch.float32 + ) + seeds = np.array([[0.0, 7.0, 7.0], [1.0, 7.0, 8.0]]) + params = nmf.init_source_params(seeds) + patches = nmf.build_a_patches(seeds, pct, sel, n_pix, d_xy=5) + a = nmf.project_spatial_profiles(params, pct, sel, n_pix, patches) + return g, a, sel, seeds + + def test_build_x_support_and_fit_phi(self): + """phi solves the per-motion least squares for structured data.""" + g, a, sel, seeds = self._setup() + n_motions = g["umyx"].shape[0] + supports = nmf.build_x_support_mots(g["h_mots"], a, sel, n_motions) + self.assertEqual(len(supports), n_motions) + + n_frames = 8 + mot_yx = np.array([0, 1] * (n_frames // 2), dtype=np.int32) + # structured data: exactly reconstructable from A with unit phi + data = torch.zeros((g["num_super_pixels"], n_frames)) + for i in range(n_motions): + frames = np.flatnonzero(mot_yx == i) + x = torch.sparse.mm(g["h_mots"][i], a[sel, :]) + data[:, frames] = x @ torch.ones((x.shape[1], len(frames))) + phi = torch.full((n_frames, seeds.shape[0]), float("nan")) + phi = nmf.fit_phi_all_motions( + a, g["h_mots"], sel, data, mot_yx, n_motions, phi + ) + # recovered phi should be ~1 for the fit frames + fit_frames = mot_yx >= 0 + np.testing.assert_allclose( + phi.numpy()[fit_frames], 1.0, rtol=1e-3, atol=1e-3 + ) + + def test_compute_source_snr_shape_and_high_for_signal(self): + """SNR is per-source and high when data is explained by the source.""" + g, a, sel, seeds = self._setup() + n_motions = g["umyx"].shape[0] + n_frames = 8 + mot_yx = np.array([0, 1] * (n_frames // 2), dtype=np.int32) + data = torch.zeros((g["num_super_pixels"], n_frames)) + phi = torch.full((n_frames, seeds.shape[0]), float("nan")) + for i in range(n_motions): + frames = np.flatnonzero(mot_yx == i) + x = torch.sparse.mm(g["h_mots"][i], a[sel, :]) + data[:, frames] = x @ torch.ones((x.shape[1], len(frames))) + phi = nmf.fit_phi_all_motions( + a, g["h_mots"], sel, data, mot_yx, n_motions, phi + ) + snr = nmf.compute_source_snr( + a, g["h_mots"], sel, data, phi, mot_yx, n_motions, seeds.shape[0] + ) + self.assertEqual(tuple(snr.shape), (2,)) + # data is fully explained -> residual ~0 -> very large SNR + self.assertTrue(torch.all(snr > 100)) + + +class TestLocalizeSources(unittest.TestCase): + """fit_sources end-to-end driver.""" + + def _inputs(self, seed=0): + """Small geometry + seeds + random residual (survives pruning).""" + g = _small_geometry(seed=seed) + npc, npr, nz = g["npc"], g["npr"], g["num_fast_zs"] + n_pix = nz * npc * npr + sel = g["sel_pix_idxs"] + coords = bg.pixel_coords_from_idxs(sel, npc, npr) + seeds = np.array([[0.0, 7.0, 7.0], [0.0, 7.0, 9.0], [1.0, 7.0, 8.0]]) + rng = np.random.default_rng(seed + 100) + n_frames = 16 + residual = rng.standard_normal( + (g["num_super_pixels"], n_frames) + ).astype(np.float32) + mot_yx = np.array([0, 1] * (n_frames // 2), dtype=np.int32) + return g, seeds, residual, mot_yx, coords, n_pix + + def test_full_run_with_pruning(self): + """Three outer iterations exercise sparsify, sort, and prune.""" + g, seeds, residual, mot_yx, coords, n_pix = self._inputs() + torch.manual_seed(7) + out = nmf.fit_sources( + seeds.copy(), + residual, + g["h_mots"], + g["umyx"], + mot_yx, + g["sel_pix_idxs"], + coords, + n_pix, + d_xy=5, + sparse_fac=float(np.exp(-3.0)), + outer_loop_iters=3, + mult_nmf_max_iters=4, + ) + self.assertEqual(out["A"].shape[0], n_pix) + self.assertEqual(out["A"].shape[1], out["n_sources"]) + self.assertEqual(out["source_params"].shape[1], 6) + self.assertEqual(out["phi_low_res"].shape[0], residual.shape[1]) + self.assertEqual(out["source_snr"].shape[0], out["n_sources"]) + self.assertGreaterEqual(out["n_sources"], 1) + + def test_deterministic_under_seed(self): + """Same torch seed -> identical spatial profiles.""" + g, seeds, residual, mot_yx, coords, n_pix = self._inputs(seed=1) + kwargs = dict( + h_mots=g["h_mots"], + unique_motion_to_keep_yx=g["umyx"], + mot_inds_yx=mot_yx, + sel_pix_idxs=g["sel_pix_idxs"], + pixel_coords=coords, + n_pixels=n_pix, + d_xy=5, + sparse_fac=float(np.exp(-3.0)), + outer_loop_iters=2, + mult_nmf_max_iters=3, + ) + torch.manual_seed(3) + a1 = nmf.fit_sources(seeds.copy(), residual, **kwargs)["A"] + torch.manual_seed(3) + a2 = nmf.fit_sources(seeds.copy(), residual, **kwargs)["A"] + np.testing.assert_array_equal(a1.numpy(), a2.numpy()) + + def test_adam_early_convergence_break(self): + """A huge gd_tol triggers the Adam convergence break.""" + g, seeds, residual, mot_yx, coords, n_pix = self._inputs(seed=2) + torch.manual_seed(5) + out = nmf.fit_sources( + seeds.copy(), + residual, + g["h_mots"], + g["umyx"], + mot_yx, + g["sel_pix_idxs"], + coords, + n_pix, + d_xy=5, + sparse_fac=float(np.exp(-3.0)), + outer_loop_iters=1, + mult_nmf_max_iters=2, + gd_tol=1e9, + ) + self.assertEqual(out["n_sources"], 3) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_bandsilo_peaks.py b/tests/test_bandsilo_peaks.py index a0a287a..c80f650 100644 --- a/tests/test_bandsilo_peaks.py +++ b/tests/test_bandsilo_peaks.py @@ -336,9 +336,11 @@ def test_exclusion_mask_3d(self): seeds = pk.get_act_im_peaks( self._act_im(), peak_th=4.0, exclusion_mask=excl, buffer_size=3 ) - for z, y, x in seeds: - if int(z) == 1: - self.assertFalse(20 <= y < 30 and 10 <= x < 20) + z1_in_box = any( + int(z) == 1 and 20 <= y < 30 and 10 <= x < 20 + for z, y, x in seeds + ) + self.assertFalse(z1_in_box) def test_all_nan_returns_empty(self): """An all-NaN activity image yields no seeds.""" diff --git a/tests/test_bandsilo_trial_data.py b/tests/test_bandsilo_trial_data.py index 657991f..4c260aa 100644 --- a/tests/test_bandsilo_trial_data.py +++ b/tests/test_bandsilo_trial_data.py @@ -159,6 +159,20 @@ def test_zero_element_line_skipped(self): # sp1 (id 101) never accumulates because its only line is empty. self.assertEqual(acc["data"][0].sum(), 0) + def test_line_with_no_matching_superpixels_skipped(self): + """A non-empty line matching none of the ids is skipped.""" + line_specs = [ + ([9], [9], [10.0]), # lookup 909 -> matches neither 101 nor 201 + ([2], [1], [20.0]), # sp2 (id 201) matches + ] + df = _MockDataFile(2, 1, line_specs, channels=1) + acc = td.accumulate_superpixel_data( + df, np.array([1.0, 2.0]), 1.0, self._super_pixel_ids(), 1, False + ) + # sp1 (101) never matches any line; sp2 (201) accumulates. + self.assertEqual(acc["data"][0].sum(), 0) + self.assertGreater(acc["data"][1].sum(), 0) + class TestReadIntegrationTrialData(unittest.TestCase): """read_integration_trial_data orchestrates open+accumulate+alignment.""" From 6c60398e2f58f95b15bfae5a693568c3be45fc1a Mon Sep 17 00:00:00 2001 From: Michael Xie Date: Mon, 27 Jul 2026 19:45:52 -0700 Subject: [PATCH 10/55] implement high-res trace extraction --- src/giant_python/bandsilo/baseline.py | 189 ++++++++ src/giant_python/bandsilo/nmf.py | 14 +- src/giant_python/bandsilo/traces.py | 612 ++++++++++++++++++++++++++ tests/test_bandsilo_baseline.py | 112 +++++ tests/test_bandsilo_traces.py | 241 ++++++++++ 5 files changed, 1164 insertions(+), 4 deletions(-) create mode 100644 src/giant_python/bandsilo/baseline.py create mode 100644 src/giant_python/bandsilo/traces.py create mode 100644 tests/test_bandsilo_baseline.py create mode 100644 tests/test_bandsilo_traces.py diff --git a/src/giant_python/bandsilo/baseline.py b/src/giant_python/bandsilo/baseline.py new file mode 100644 index 0000000..38e5ea8 --- /dev/null +++ b/src/giant_python/bandsilo/baseline.py @@ -0,0 +1,189 @@ +"""Baseline (F0) estimation and dF/F assembly. + +Ported from ``extractSLAP2IntegrationSources.py`` (``_movmean_nan`` / +``computeF0`` ref 217-305, and the dF/F assembly ref 2972-2978). +:func:`compute_f0` estimates a slowly-varying fluorescence baseline via a +rolling median plus a decimated-grid rolling-min ("convex-hull"-like) +envelope, interpolated back to the full time grid with PCHIP. +:func:`assemble_dff` combines the per-trial +least-squares traces into ``F``, re-estimates ``F0``, and forms ``dF``/``dFF``. +""" + +from __future__ import annotations + +from typing import Tuple + +import numpy as np +import pandas as pd +from scipy.interpolate import PchipInterpolator + + +def _movmean_nan(x: np.ndarray, window: int) -> np.ndarray: + """Centered, NaN-ignoring moving mean (like MATLAB ``smoothdata movmean``). + + Uses partial windows at the edges and returns NaN where a window contains + no valid samples. + + Parameters + ---------- + x : ndarray + 1-D input (NaNs allowed). + window : int + Window length in samples (clamped to >= 1). + + Returns + ------- + ndarray of float + The smoothed signal. + """ + window = max(int(window), 1) + if window == 1: + return x.astype(float, copy=True) + + kernel = np.ones(window, dtype=float) + valid = ~np.isnan(x) + x_filled = np.where(valid, x, 0.0) + + num = np.convolve(x_filled, kernel, mode="same") + den = np.convolve(valid.astype(float), kernel, mode="same") + + return num / np.where(den == 0.0, np.nan, den) + + +def _compute_f0_column( + f0_col: np.ndarray, + sample_times: np.ndarray, + n_samps_in_hull: int, + total_frames: int, +) -> np.ndarray: + """Estimate the baseline for one already-median-filtered trace column. + + Builds a set of decimated-grid interpolations of the trace, takes their + per-sample minimum (the rolling-min envelope), discards samples with too + few contributions, smooths/fills the gaps, and PCHIP-interpolates the + envelope back onto the full ``total_frames`` grid. + """ + f00 = np.full((sample_times.shape[0], n_samps_in_hull), np.nan) + for dix in range(n_samps_in_hull, 0, -1): + start = dix - 1 + xi = sample_times[start::n_samps_in_hull] + f00[:, dix - 1] = np.interp( + sample_times, xi, f0_col[xi], left=np.nan, right=np.nan + ) + ff = np.nanmin(f00, axis=1) + + doubt = np.sum(~np.isnan(f00), axis=1) < int(np.ceil(n_samps_in_hull / 2)) + if np.sum(~doubt) > 2: + ff[doubt] = np.nan + + win = 2 * int(np.ceil(n_samps_in_hull / 2.0)) + 1 + fill = _movmean_nan(ff, win) + nan_mask = np.isnan(ff) + ff[nan_mask] = fill[nan_mask] + ff = _movmean_nan(ff, win) + + nan_mask = np.isnan(ff) + if np.any(nan_mask): + ff = np.interp(sample_times, sample_times[~nan_mask], ff[~nan_mask]) + + pchip = PchipInterpolator(sample_times, ff, extrapolate=True) + return pchip(np.arange(total_frames)) + + +def compute_f0( + f_in: np.ndarray, denoise_window: int, hull_window: int +) -> np.ndarray: + """Estimate a slowly-varying fluorescence baseline ``F0``. + + Port of ``computeF0``: (1) a centered rolling-median denoise, (2) a + rolling convex-hull-like min envelope on decimated grids, (3) discard of + doubtful samples near NaNs, smoothing/filling, and PCHIP back to the full + time grid. + + Parameters + ---------- + f_in : ndarray of shape (T, ...) + Fluorescence with time along axis 0 (NaNs allowed). + denoise_window : int + Rolling-median window (time samples). + hull_window : int + Window controlling the convex-hull-like envelope. + + Returns + ------- + ndarray + The baseline ``F0``, same shape as ``f_in``. + """ + f = np.asarray(f_in) + orig_shape = f.shape + if f.ndim == 1: + f = f[:, None] + else: + f = f.reshape(f.shape[0], -1) + + total_frames = f.shape[0] + if total_frames < 4: + return np.ones_like(f_in, dtype=float) * np.nanmean( + f_in, axis=0, keepdims=True + ) + + hull_window = int(min(hull_window, total_frames // 4)) + delta_des = max(4.0, denoise_window / 6.0) + + sample_times = np.rint( + np.linspace( + 0, total_frames - 1, num=int(np.ceil(total_frames / delta_des) + 1) + ) + ).astype(int) + n_samps_in_hull = int(np.ceil(hull_window / delta_des)) + + f0 = ( + pd.DataFrame(f) + .rolling(window=denoise_window, center=True, min_periods=1) + .median() + .to_numpy() + ) + + f0 = f0.reshape(f0.shape[0], -1) + for cix in range(f0.shape[1]): + if np.all(np.isnan(f0[:, cix])): + continue + f0[:, cix] = _compute_f0_column( + f0[:, cix], sample_times, n_samps_in_hull, total_frames + ) + + return f0.reshape(orig_shape) + + +def assemble_dff( + d_f_ls: np.ndarray, + f0_ls: np.ndarray, + denoise_window: int, + hull_window: int, +) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """Assemble ``F``, re-estimate ``F0``, and form ``dF``/``dFF``. + + Reconstructs total fluorescence ``F = dF_ls + F0_ls`` from the per-motion + least-squares source and background traces, re-estimates the baseline with + :func:`compute_f0`, and computes ``dF = F - F0`` and ``dFF = dF / + clip(F0, 1e-4, inf)``. + + Parameters + ---------- + d_f_ls : ndarray of shape (T, n_sources) + Concatenated per-source least-squares dF traces (``phi``). + f0_ls : ndarray of shape (T, n_sources) + Concatenated per-source least-squares background traces. + denoise_window, hull_window : int + Windows passed to :func:`compute_f0`. + + Returns + ------- + f, f0, d_f, d_ff : ndarray + Total fluorescence, baseline, baseline-subtracted, and dF/F. + """ + f = d_f_ls + f0_ls + f0 = compute_f0(f, denoise_window, hull_window) + d_f = f - f0 + d_ff = d_f / np.clip(f0, 1e-4, None) + return f, f0, d_f, d_ff diff --git a/src/giant_python/bandsilo/nmf.py b/src/giant_python/bandsilo/nmf.py index 55ffd43..0c6f655 100644 --- a/src/giant_python/bandsilo/nmf.py +++ b/src/giant_python/bandsilo/nmf.py @@ -262,10 +262,16 @@ def _motion_frames(mot_inds_yx: np.ndarray, i: int) -> np.ndarray: return np.flatnonzero(mot_inds_yx == i) -def _solve_phi_motion( +def solve_phi_motion( x: torch.Tensor, data_motion: torch.Tensor ) -> torch.Tensor: - """Least-squares temporal weights for one motion's superpixel profiles.""" + """Least-squares temporal weights for one motion's superpixel profiles. + + Solves the regularized normal equations + ``(XᵀX + 1e-10 I) phi = Xᵀ data`` and returns ``phiᵀ`` (frames x sources). + Shared by the NMF fits here and the high-res trace solve in + :mod:`giant_python.bandsilo.traces`. + """ xtx = x.T @ x xtd = x.T @ data_motion regularized = xtx + 1e-10 * torch.eye(xtx.shape[0]) @@ -308,7 +314,7 @@ def fit_phi_all_motions( for i in range(n_motions): frames = _motion_frames(mot_inds_yx, i) x = torch.sparse.mm(h_mots[i], a[sel_pix_idxs, :]) - phi_low_res[frames, :] = _solve_phi_motion(x, data_for_nmf[:, frames]) + phi_low_res[frames, :] = solve_phi_motion(x, data_for_nmf[:, frames]) return phi_low_res @@ -367,7 +373,7 @@ def multiplicative_nmf( x[~x_support_mots[i]] = 0 x = x / torch.norm(x, dim=0, keepdim=True) - phi = torch.clamp(_solve_phi_motion(x, data_for_nmf[:, frames]), min=0) + phi = torch.clamp(solve_phi_motion(x, data_for_nmf[:, frames]), min=0) for iter_idx in range(max_iters): numerator = x.T @ data_for_nmf[:, frames] diff --git a/src/giant_python/bandsilo/traces.py b/src/giant_python/bandsilo/traces.py new file mode 100644 index 0000000..ceaefb6 --- /dev/null +++ b/src/giant_python/bandsilo/traces.py @@ -0,0 +1,612 @@ +"""High-resolution per-trial trace extraction. + +Ported from ``get_high_res_traces`` in +``extractSLAP2IntegrationSources.py`` (ref 844-1011). For one trial this reads +the high-res superpixel activity, interpolates the alignment motion/background +onto the trial's downsample grid, bins the frames by motion (matching the +low-res motion bins used during NMF), and — per motion bin — projects the fixed +spatial profiles ``A_final`` into superpixel space and least-squares-solves the +per-source temporal traces (``phi``, the least-squares dF) and the background +projection (``F0``). It also computes the global and per-user-ROI fluorescence. + +The pure compute (:func:`compute_high_res_traces`) is factored out of the IO +wrapper (:func:`get_high_res_traces`, which reads the SLAP2 file lazily via +:mod:`giant_python.bandsilo.trial_data`) so the numerics are testable without +``slap2_utils``. Per-trial ``.npz`` caching and the ``mp.Pool`` fan-out are +deferred to the Phase-8 pipeline driver. +""" + +from __future__ import annotations + +from typing import List, Optional, Tuple + +import numpy as np +import torch + +from .background import build_motion_h_matrices, build_selected_pixel_mask +from .geometry import ref_pixs_to_drc +from .nmf import solve_phi_motion +from .trial_data import nearest_interp, read_integration_trial_data + + +def _empty_trace_result( + n_sources: int, num_channels: int, n_soma: int +) -> tuple: + """Return the all-empty result tuple for a skipped trial. + + Parameters + ---------- + n_sources : int + Number of sources (column count of the empty ``phi``/``F0``). + num_channels : int + Number of acquisition channels. + n_soma : int + Number of user/soma ROIs. + + Returns + ------- + tuple + Same 8-element shape as :func:`compute_high_res_traces`, with zero + frames. + """ + return ( + np.full((0, n_sources), np.nan, dtype=np.float32), + np.full((0, n_sources), np.nan, dtype=np.float32), + np.full((0,), np.nan), + np.full((0,), np.nan), + np.full((0, num_channels), np.nan, dtype=np.float32), + (np.full((0,), np.nan), np.full((0,), np.nan), np.full((0,), np.nan)), + ( + np.full((0,), 0, dtype=np.int16), + np.full((0,), 0, dtype=np.int16), + np.full((0,), 0, dtype=np.int16), + ), + np.full((0, n_soma, num_channels), np.nan, dtype=np.float32), + ) + + +def _interp_trial_alignment( + frames: np.ndarray, a_data: dict, background_ds: np.ndarray +): + """Interpolate alignment motion/background onto the trial's frame grid. + + Motion and per-superpixel background are linearly interpolated from the + alignment downsample grid to ``frames``; the online shifts use + nearest-neighbor interpolation. + + Parameters + ---------- + frames : ndarray + The trial's high-res downsample frame line positions. + a_data : dict + Alignment data (``DSframes``, ``motionDSr/c/z``, + ``onlineYshift/Xshift/Zshift``). + background_ds : ndarray of shape (n_superpixels, n_ds) + Per-superpixel low-res background for this trial. + + Returns + ------- + tuple + ``(motion_r, motion_c, motion_z, background, online_y, online_x, + online_z)``. + """ + ds = a_data["DSframes"] + motion_r = np.interp(frames, ds, a_data["motionDSr"]) + motion_c = np.interp(frames, ds, a_data["motionDSc"]) + motion_z = np.interp(frames, ds, a_data["motionDSz"]) + background = np.array( + [ + np.interp(frames, ds, background_ds[i]) + for i in range(background_ds.shape[0]) + ] + ) + online_y = nearest_interp(frames, ds, a_data["onlineYshift"]) + online_x = nearest_interp(frames, ds, a_data["onlineXshift"]) + online_z = nearest_interp(frames, ds, a_data["onlineZshift"]) + return ( + motion_r, + motion_c, + motion_z, + background, + online_y, + online_x, + online_z, + ) + + +def _bin_trial_motion( + motion_r: np.ndarray, + motion_c: np.ndarray, + motion_z: np.ndarray, + median_z: float, + unique_motion_ds: np.ndarray, + mot_inds_to_keep_ds: np.ndarray, +) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """Bin trial frames by motion and match them to the low-res motion bins. + + Frames within 1.5 um of the median z are rebinned into 2-D (row, col) + motion bins; each retained low-res bin is matched to its trial bin so the + same PSF geometry is reused. + + Parameters + ---------- + motion_r, motion_c, motion_z : ndarray + Per-frame interpolated motion. + median_z : float + Median z across the recording (from the low-res pass). + unique_motion_ds : ndarray of shape (n_ds_bins, 2+) + The low-res kept 2-D motion vectors. + mot_inds_to_keep_ds : ndarray + Indices of the low-res bins to keep. + + Returns + ------- + unique_motion : ndarray of shape (n_bins, 2) + Trial 2-D motion bins (over the z-kept frames). + mot_inds : ndarray of shape (n_frames,), int32 + Per-frame trial-bin index (``-1`` for dropped frames). + mot_inds_to_keep : ndarray + Trial-bin indices matching a kept low-res bin. + frames_to_keep : ndarray of bool + Frames belonging to a matched bin. + """ + unique_motion3, mot_inds = np.unique( + np.round(np.stack((motion_r, motion_c, motion_z), axis=1)), + axis=0, + return_inverse=True, + ) + mot_inds = np.reshape(mot_inds, -1) + frames_to_keep = np.isin( + mot_inds, + np.flatnonzero(np.abs(unique_motion3[:, 2] - median_z) <= 1.5), + ) + + mot_inds = -1 * np.ones((len(mot_inds),), dtype=np.int32) + stacked = np.round( + np.stack((motion_r, motion_c), axis=1)[frames_to_keep, :] + ) + unique_motion, inv = np.unique(stacked, axis=0, return_inverse=True) + mot_inds[frames_to_keep] = np.reshape(inv, -1) + + mot_inds_to_keep = -1 * np.ones_like(mot_inds_to_keep_ds, dtype=np.int64) + for i, motion_idx_ds in enumerate(mot_inds_to_keep_ds): + matches = np.flatnonzero( + np.all( + unique_motion[:, :2] == unique_motion_ds[motion_idx_ds, :2], + axis=1, + ) + ) + if len(matches) > 0: + mot_inds_to_keep[i] = matches[0] + mot_inds_to_keep = mot_inds_to_keep[mot_inds_to_keep != -1] + + frames_to_keep = np.isin(mot_inds, mot_inds_to_keep) + return unique_motion, mot_inds, mot_inds_to_keep, frames_to_keep + + +def _solve_trial_phi_f0( + data: np.ndarray, + background: np.ndarray, + unique_motion: np.ndarray, + mot_inds: np.ndarray, + mot_inds_to_keep: np.ndarray, + sparse_h_inds: np.ndarray, + sparse_h_vals: np.ndarray, + sel_pix_idxs: np.ndarray, + a_final: torch.Tensor, + num_super_pixels: int, + dmd_pixels_per_row: int, +) -> Tuple[torch.Tensor, torch.Tensor]: + """Least-squares-solve per-source traces and background projection. + + For each kept motion bin, shifts ``H`` by the bin's motion, projects + ``A_final`` into superpixel space, and solves for the source temporal + weights (from the background-subtracted residual) and the background + projection ``F0``. + + Parameters + ---------- + data : ndarray of shape (n_superpixels, n_frames) + Count-normalized activity. + background : ndarray of shape (n_superpixels, n_frames) + Interpolated background. + unique_motion : ndarray of shape (n_bins, 2) + Trial 2-D motion bins. + mot_inds : ndarray of shape (n_frames,) + Per-frame trial-bin index. + mot_inds_to_keep : ndarray + Trial-bin indices to solve. + sparse_h_inds, sparse_h_vals : ndarray + Base sparse ``H`` (from ``geometry.build_sparse_h``). + sel_pix_idxs : ndarray of int + Sorted flat selected-pixel indices. + a_final : torch.Tensor of shape (n_pixels, n_sources) + Fixed source spatial profiles. + num_super_pixels : int + Superpixel count (``H`` row count). + dmd_pixels_per_row : int + Grid geometry (column-shift stride). + + Returns + ------- + phi, f0 : torch.Tensor of shape (n_frames, n_sources) + Per-source least-squares dF and background projection (NaN where no + motion bin applies). + """ + n_sources = a_final.shape[1] + n_frames = data.shape[1] + phi = torch.full((n_frames, n_sources), float("nan"), dtype=torch.float32) + f0 = torch.full((n_frames, n_sources), float("nan"), dtype=torch.float32) + residual = data - background + + # One shifted superpixel<-pixel H per trial motion bin (same builder the + # NMF pass uses); only the matched bins are solved below. + h_mots = build_motion_h_matrices( + sparse_h_inds, + sparse_h_vals, + unique_motion, + sel_pix_idxs, + num_super_pixels, + dmd_pixels_per_row, + ) + + for motion_idx in mot_inds_to_keep: + motion_frames = np.flatnonzero(mot_inds == motion_idx) + x = torch.sparse.mm(h_mots[motion_idx], a_final[sel_pix_idxs, :]) + # phi from the background-subtracted residual; F0 from the background, + # both via the shared regularized normal-equations solve. + phi[motion_frames, :] = solve_phi_motion( + x, torch.from_numpy(residual[:, motion_frames].astype(np.float32)) + ) + f0[motion_frames, :] = solve_phi_motion( + x, + torch.from_numpy(background[:, motion_frames].astype(np.float32)), + ) + + return phi, f0 + + +def _compute_global_f( + data: np.ndarray, + data2: Optional[np.ndarray], + frames_to_keep: np.ndarray, + num_channels: int, +) -> np.ndarray: + """Sum fluorescence over all superpixels per channel and frame. + + Parameters + ---------- + data : ndarray of shape (n_superpixels, n_frames) + Channel-1 activity. + data2 : ndarray or None + Channel-2 activity (used when ``num_channels >= 2``). + frames_to_keep : ndarray of bool + Dropped frames are set to NaN. + num_channels : int + Number of acquisition channels. + + Returns + ------- + ndarray of shape (n_frames, num_channels) + Global fluorescence. + """ + channel_sums = [np.sum(data, axis=0)] + if num_channels >= 2: + channel_sums.append(np.sum(data2, axis=0)) + global_f = np.stack(channel_sums, axis=-1) + global_f[~frames_to_keep, :] = np.nan + return global_f + + +def _compute_f_soma( + data: np.ndarray, + data2: Optional[np.ndarray], + soma_sps: List[np.ndarray], + frames_to_keep: np.ndarray, + num_channels: int, +) -> np.ndarray: + """Sum fluorescence over each user-ROI's superpixels per channel. + + Parameters + ---------- + data : ndarray of shape (n_superpixels, n_frames) + Channel-1 activity. + data2 : ndarray or None + Channel-2 activity (used when ``num_channels >= 2``). + soma_sps : list of ndarray + Per-ROI superpixel index arrays. + frames_to_keep : ndarray of bool + Dropped frames are set to NaN. + num_channels : int + Number of acquisition channels. + + Returns + ------- + ndarray of shape (n_frames, n_rois, num_channels) + Per-ROI fluorescence. + """ + n_frames = data.shape[1] + f_soma = np.full( + (n_frames, len(soma_sps), num_channels), np.nan, dtype=np.float32 + ) + for i, roi_sps in enumerate(soma_sps): + f_soma[:, i, 0] = np.nansum(data[roi_sps, :], axis=0) + if num_channels >= 2: + f_soma[:, i, 1] = np.nansum(data2[roi_sps, :], axis=0) + f_soma[~frames_to_keep, :, :] = np.nan + return f_soma + + +def compute_high_res_traces( + data: np.ndarray, + data2: Optional[np.ndarray], + background_ds: np.ndarray, + frames: np.ndarray, + a_data: dict, + subsample_matrix_inds: np.ndarray, + sparse_h_inds: np.ndarray, + sparse_h_vals: np.ndarray, + a_final: torch.Tensor, + unique_motion_ds: np.ndarray, + mot_inds_to_keep_ds: np.ndarray, + median_z: float, + psf2d: np.ndarray, + num_super_pixels: int, + num_fast_zs: int, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, + num_channels: int, + soma_sps: List[np.ndarray], +) -> tuple: + """Compute one trial's high-res source traces from loaded arrays. + + Parameters + ---------- + data : ndarray of shape (n_superpixels, n_frames) + Count-normalized channel-1 activity. + data2 : ndarray or None + Count-normalized channel-2 activity. + background_ds : ndarray of shape (n_superpixels, n_ds) + Per-superpixel low-res background for this trial. + frames : ndarray + The trial's downsample frame line positions. + a_data : dict + Alignment data for this trial. + subsample_matrix_inds : ndarray + Superpixel reference-pixel map (from ``geometry``). + sparse_h_inds, sparse_h_vals : ndarray + Base sparse ``H``. + a_final : torch.Tensor of shape (n_pixels, n_sources) + Fixed source spatial profiles. + unique_motion_ds : ndarray + Low-res kept 2-D motion vectors. + mot_inds_to_keep_ds : ndarray + Indices of the low-res bins to keep. + median_z : float + Median z (from the low-res pass). + psf2d : ndarray + The (cropped) PSF for this DMD. + num_super_pixels, num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row \ +: int + Geometry. + num_channels : int + Number of acquisition channels. + soma_sps : list of ndarray + Per-ROI superpixel index arrays. + + Returns + ------- + tuple + ``(phi, F0, frames, sel_pix_idxs, global_f, (motion_r, motion_c, + motion_z), (online_y, online_x, online_z), F_soma)``. ``phi`` and + ``F0`` are ``(n_frames, n_sources)`` ndarrays. + """ + ( + motion_r, + motion_c, + motion_z, + background, + online_y, + online_x, + online_z, + ) = _interp_trial_alignment(frames, a_data, background_ds) + + ( + unique_motion, + mot_inds, + mot_inds_to_keep, + frames_to_keep, + ) = _bin_trial_motion( + motion_r, + motion_c, + motion_z, + median_z, + unique_motion_ds, + mot_inds_to_keep_ds, + ) + + ref_d, ref_c, ref_r = ref_pixs_to_drc( + subsample_matrix_inds[:, 0], dmd_pixels_per_column, dmd_pixels_per_row + ) + _, sel_pix_idxs = build_selected_pixel_mask( + unique_motion, + ref_d, + ref_r, + ref_c, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + psf2d, + ) + + phi, f0 = _solve_trial_phi_f0( + data, + background, + unique_motion, + mot_inds, + mot_inds_to_keep, + sparse_h_inds, + sparse_h_vals, + sel_pix_idxs, + a_final, + num_super_pixels, + dmd_pixels_per_row, + ) + + global_f = _compute_global_f(data, data2, frames_to_keep, num_channels) + f_soma = _compute_f_soma( + data, data2, soma_sps, frames_to_keep, num_channels + ) + + return ( + phi.numpy(), + f0.numpy(), + frames, + sel_pix_idxs, + global_f, + (motion_r, motion_c, motion_z), + (online_y, online_x, online_z), + f_soma, + ) + + +def _load_high_res_trial_data( + trial_ix: int, + dmd_ix: int, + samp_freq: float, + super_pixel_ids: np.ndarray, + datadr: str, + trial_table: dict, + num_channels: int, +): # pragma: no cover - requires slap2_utils + """Read one trial's high-res activity via the SLAP2 reader. + + Returns + ------- + tuple + ``(data, data2, a_data, frames)`` with count-normalized activity. + """ + result = read_integration_trial_data( + trial_ix, + True, + dmd_ix, + samp_freq, + super_pixel_ids, + datadr, + trial_table, + num_channels, + all_channels=True, + ) + data = result["data"] / result["data_count"] + data2 = None + if num_channels >= 2: + data2 = result["data2"] / result["data_count2"] + return data, data2, result["alignment"], result["ds_frames"] + + +def get_high_res_traces( + trial_info: tuple, + dmd_ix: int, + samp_freq: float, + super_pixel_ids: np.ndarray, + datadr: str, + trial_table: dict, + subsample_matrix_inds: np.ndarray, + sparse_h_inds: np.ndarray, + sparse_h_vals: np.ndarray, + a_final, + unique_motion_ds: np.ndarray, + mot_inds_to_keep_ds: np.ndarray, + median_z: float, + psf2d: np.ndarray, + num_super_pixels: int, + num_fast_zs: int, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, + num_channels: int, + soma_sps: List[np.ndarray], +) -> tuple: + """Read and extract one trial's high-res source traces. + + Skipped trials (``keep_trial`` False) return the empty result immediately; + otherwise the trial is read (via ``slap2_utils``) and passed to + :func:`compute_high_res_traces`. + + Parameters + ---------- + trial_info : tuple + ``(trial_ix, keep_trial, background_ds)`` for this trial. + dmd_ix : int + 0-based DMD index. + samp_freq : float + Analysis sample rate, in Hz. + super_pixel_ids : ndarray + Superpixel id lookup for this DMD. + datadr : str + Raw-data directory. + trial_table : dict + Normalized trial table. + subsample_matrix_inds, sparse_h_inds, sparse_h_vals : ndarray + Geometry / sparse ``H``. + a_final : ndarray or torch.Tensor + Fixed source spatial profiles ``(n_pixels, n_sources)``. + unique_motion_ds, mot_inds_to_keep_ds : ndarray + Low-res motion bins / kept indices. + median_z : float + Median z (from the low-res pass). + psf2d : ndarray + The (cropped) PSF for this DMD. + num_super_pixels, num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row \ +: int + Geometry. + num_channels : int + Number of acquisition channels. + soma_sps : list of ndarray + Per-ROI superpixel index arrays. + + Returns + ------- + tuple + The 8-element result of :func:`compute_high_res_traces` (empty for + skipped trials). + """ + trial_ix, keep_trial, background_ds = trial_info + + if not isinstance(a_final, torch.Tensor): + a_final = torch.tensor(a_final, dtype=torch.float32) + n_sources = a_final.shape[1] + + if not keep_trial: + return _empty_trace_result(n_sources, num_channels, len(soma_sps)) + + data, data2, a_data, frames = _load_high_res_trial_data( + trial_ix, + dmd_ix, + samp_freq, + super_pixel_ids, + datadr, + trial_table, + num_channels, + ) + + return compute_high_res_traces( + data, + data2, + background_ds, + frames, + a_data, + subsample_matrix_inds, + sparse_h_inds, + sparse_h_vals, + a_final, + unique_motion_ds, + mot_inds_to_keep_ds, + median_z, + psf2d, + num_super_pixels, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + num_channels, + soma_sps, + ) diff --git a/tests/test_bandsilo_baseline.py b/tests/test_bandsilo_baseline.py new file mode 100644 index 0000000..1caf62a --- /dev/null +++ b/tests/test_bandsilo_baseline.py @@ -0,0 +1,112 @@ +"""Tests for giant_python.bandsilo.baseline (Phase 7 F0 / dF-F). + +Covers the NaN-aware moving mean, the rolling-min-envelope ``compute_f0`` +baseline estimator, and the dF/F assembly. ``compute_f0`` is additionally +cross-checked against a verbatim copy of the reference in the project's +development notes; here we assert structural properties on synthetic traces. +""" + +import unittest + +import numpy as np + +from giant_python.bandsilo import baseline as bl + + +class TestMovmeanNan(unittest.TestCase): + """_movmean_nan edge cases.""" + + def test_window_one_is_identity(self): + """A window of 1 returns a float copy of the input.""" + x = np.array([1.0, np.nan, 3.0]) + out = bl._movmean_nan(x, 1) + np.testing.assert_array_equal( + np.nan_to_num(out), np.nan_to_num(x.astype(float)) + ) + + def test_ignores_nan(self): + """NaNs are excluded from the mean, not treated as zero.""" + x = np.array([2.0, np.nan, 4.0]) + out = bl._movmean_nan(x, 3) + # center window over index 1 sees {2, 4} -> mean 3 + self.assertAlmostEqual(out[1], 3.0) + + def test_all_nan_yields_nan(self): + """A window with no valid samples yields NaN (den == 0 branch).""" + x = np.full(5, np.nan) + out = bl._movmean_nan(x, 3) + self.assertTrue(np.all(np.isnan(out))) + + +class TestComputeF0(unittest.TestCase): + """compute_f0 shape handling and baseline behavior.""" + + def _drifting_trace(self, total=200, seed=0): + """A drifting baseline plus positive transients and noise.""" + rng = np.random.default_rng(seed) + t = np.arange(total) + base = 5 + 2 * np.sin(t / 40) + transients = np.clip(rng.standard_normal(total), 0, None) * 3 + return (base + transients + rng.standard_normal(total) * 0.2).astype( + float + ) + + def test_tracks_below_signal(self): + """F0 stays at or below the (transient-carrying) signal.""" + f = self._drifting_trace() + f0 = bl.compute_f0(f, 9, 40) + self.assertEqual(f0.shape, f.shape) + # baseline should sit near the lower envelope, well below the mean + self.assertLess(np.nanmean(f0), np.nanmean(f)) + + def test_two_dimensional(self): + """A 2-D input is handled column-wise and keeps its shape.""" + f = np.stack( + [self._drifting_trace(seed=1), self._drifting_trace(seed=2)], + axis=1, + ) + f[10:15, 0] = np.nan + f0 = bl.compute_f0(f, 9, 40) + self.assertEqual(f0.shape, f.shape) + + def test_all_nan_column_skipped(self): + """An all-NaN column is returned unchanged (all NaN).""" + f = np.stack( + [self._drifting_trace(seed=3), np.full(200, np.nan)], axis=1 + ) + f0 = bl.compute_f0(f, 9, 40) + self.assertTrue(np.all(np.isnan(f0[:, 1]))) + self.assertFalse(np.all(np.isnan(f0[:, 0]))) + + def test_short_input_returns_mean(self): + """Fewer than 4 samples returns the broadcast column mean.""" + f = np.array([[2.0, 8.0], [4.0, 10.0], [6.0, 12.0]]) + f0 = bl.compute_f0(f, 9, 40) + np.testing.assert_allclose(f0[:, 0], 4.0) + np.testing.assert_allclose(f0[:, 1], 10.0) + + def test_sparse_hull_triggers_gap_interpolation(self): + """A trace with NaN gaps exercises the residual-NaN interpolation.""" + f = self._drifting_trace(total=120, seed=4) + f[30:70] = np.nan # a wide gap forces envelope NaNs to be filled + f0 = bl.compute_f0(f, 15, 80) + self.assertEqual(f0.shape, f.shape) + self.assertFalse(np.all(np.isnan(f0))) + + +class TestAssembleDff(unittest.TestCase): + """assemble_dff combines the least-squares traces into dF/F.""" + + def test_relationships(self): + """F = dF_ls + F0_ls; dF = F - F0; dFF = dF / clip(F0).""" + rng = np.random.default_rng(0) + d_f_ls = np.clip(rng.standard_normal((150, 2)), 0, None) * 2 + f0_ls = np.full((150, 2), 5.0) + rng.standard_normal((150, 2)) * 0.1 + f, f0, d_f, d_ff = bl.assemble_dff(d_f_ls, f0_ls, 9, 40) + np.testing.assert_allclose(f, d_f_ls + f0_ls) + np.testing.assert_allclose(d_f, f - f0) + np.testing.assert_allclose(d_ff, d_f / np.clip(f0, 1e-4, None)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_bandsilo_traces.py b/tests/test_bandsilo_traces.py new file mode 100644 index 0000000..74a758b --- /dev/null +++ b/tests/test_bandsilo_traces.py @@ -0,0 +1,241 @@ +"""Tests for giant_python.bandsilo.traces (Phase 7 high-res traces). + +Covers the per-trial compute (motion interp/binning, per-motion least-squares +``phi``/``F0``, global and per-ROI fluorescence) and the IO wrapper's +skip/monkeypatched-read paths. ``compute_high_res_traces`` is additionally +cross-checked against a verbatim copy of the reference in the project's +development notes; here we assert structural behavior on synthetic data. +""" + +import unittest + +import numpy as np +import torch + +from giant_python.bandsilo import geometry as geo +from giant_python.bandsilo import traces as tr + + +def _geometry(npc=15, npr=15, num_fast_zs=2): + """Build a small consistent geometry (subsample map + sparse H + PSF).""" + plane = npc * npr + positions = [(r, c) for r in range(4, 11) for c in range(4, 11)] + rows = [] + for z in range(num_fast_zs): + for r, c in positions: + rows.append([z * plane + r * npr + c, len(rows) + 1]) + smi = np.array(rows, dtype=np.int32) + yy, xx = np.mgrid[-2:3, -2:3] + psf2d = np.exp(-(yy**2 + xx**2) / 2.0).astype(np.float32) + sh_inds, sh_vals = geo.build_sparse_h(smi, psf2d, npc, npr) + return dict( + npc=npc, + npr=npr, + nz=num_fast_zs, + nsp=smi.shape[0], + smi=smi, + psf2d=psf2d, + sh_inds=sh_inds, + sh_vals=sh_vals, + ) + + +def _trial_inputs(num_channels=1, unique_motion_ds=None, seed=2): + """Synthetic loaded-trial arrays + geometry for compute_high_res_traces.""" + g = _geometry() + rng = np.random.default_rng(seed) + nsp = g["nsp"] + n_ds = 60 + n_frames = 40 + ds_frames = np.arange(n_ds, dtype=float) + frames = np.linspace(1, n_ds - 2, n_frames) + a_data = dict( + DSframes=ds_frames, + motionDSr=(np.arange(n_ds) % 2).astype(float), # bins 0 and 1 + motionDSc=np.zeros(n_ds), + motionDSz=np.zeros(n_ds), + onlineYshift=rng.standard_normal(n_ds), + onlineXshift=rng.standard_normal(n_ds), + onlineZshift=rng.standard_normal(n_ds), + ) + data = rng.standard_normal((nsp, n_frames)).astype(np.float32) + 2 + data2 = ( + rng.standard_normal((nsp, n_frames)).astype(np.float32) + 1 + if num_channels >= 2 + else None + ) + background_ds = rng.standard_normal((nsp, n_ds)).astype(np.float32) + 1 + if unique_motion_ds is None: + unique_motion_ds = np.array([[0, 0], [1, 0]], dtype=float) + mot_keep_ds = np.arange(unique_motion_ds.shape[0]) + n_pixels = g["nz"] * g["npc"] * g["npr"] + a_final = torch.rand((n_pixels, 3), dtype=torch.float32) + soma_sps = [np.array([0, 1, 2]), np.array([5, 6, 7, 8])] + return dict( + g=g, + data=data, + data2=data2, + background_ds=background_ds, + frames=frames, + a_data=a_data, + a_final=a_final, + unique_motion_ds=unique_motion_ds, + mot_keep_ds=mot_keep_ds, + soma_sps=soma_sps, + num_channels=num_channels, + n_frames=n_frames, + ) + + +def _compute(inp): + """Call compute_high_res_traces with the assembled inputs dict.""" + g = inp["g"] + return tr.compute_high_res_traces( + inp["data"], + inp["data2"], + inp["background_ds"], + inp["frames"], + inp["a_data"], + g["smi"], + g["sh_inds"], + g["sh_vals"], + inp["a_final"], + inp["unique_motion_ds"], + inp["mot_keep_ds"], + 0.0, + g["psf2d"], + g["nsp"], + g["nz"], + g["npc"], + g["npr"], + inp["num_channels"], + inp["soma_sps"], + ) + + +class TestComputeHighResTraces(unittest.TestCase): + """compute_high_res_traces per-trial numerics.""" + + def test_single_channel_shapes_and_fit(self): + """Returns the 8-tuple; phi/F0 are per-frame per-source with fits.""" + inp = _trial_inputs(num_channels=1) + out = _compute(inp) + phi, f0, frames, sel, global_f, motion, online, f_soma = out + n_sources = inp["a_final"].shape[1] + self.assertEqual(phi.shape, (inp["n_frames"], n_sources)) + self.assertEqual(f0.shape, (inp["n_frames"], n_sources)) + self.assertEqual(frames.shape, (inp["n_frames"],)) + self.assertEqual(global_f.shape, (inp["n_frames"], 1)) + self.assertEqual( + f_soma.shape, (inp["n_frames"], len(inp["soma_sps"]), 1) + ) + self.assertEqual(len(motion), 3) + self.assertEqual(len(online), 3) + self.assertTrue(np.any(np.isfinite(phi))) + + def test_two_channels(self): + """A second channel adds columns to global and soma fluorescence.""" + inp = _trial_inputs(num_channels=2) + out = _compute(inp) + _, _, _, _, global_f, _, _, f_soma = out + self.assertEqual(global_f.shape, (inp["n_frames"], 2)) + self.assertEqual( + f_soma.shape, (inp["n_frames"], len(inp["soma_sps"]), 2) + ) + # both channels populated where frames are kept + self.assertTrue(np.any(np.isfinite(f_soma[:, :, 1]))) + + def test_unmatched_ds_bin_skipped(self): + """A low-res motion bin absent from the trial is skipped.""" + umd = np.array([[0, 0], [1, 0], [5, 5]], dtype=float) # [5,5] absent + inp = _trial_inputs(num_channels=1, unique_motion_ds=umd) + out = _compute(inp) + # still produces finite fits for the two present bins + self.assertTrue(np.any(np.isfinite(out[0]))) + + +class TestGetHighResTraces(unittest.TestCase): + """get_high_res_traces IO wrapper (skip + monkeypatched read).""" + + def test_skipped_trial_returns_empty(self): + """keep_trial=False returns the empty 8-tuple without reading.""" + inp = _trial_inputs(num_channels=2) + g = inp["g"] + out = tr.get_high_res_traces( + (0, False, inp["background_ds"]), + 0, + 100.0, + np.zeros((g["nsp"], 1), dtype=np.int32), + "unused", + {}, + g["smi"], + g["sh_inds"], + g["sh_vals"], + inp["a_final"].numpy(), + inp["unique_motion_ds"], + inp["mot_keep_ds"], + 0.0, + g["psf2d"], + g["nsp"], + g["nz"], + g["npc"], + g["npr"], + 2, + inp["soma_sps"], + ) + self.assertEqual(out[0].shape, (0, inp["a_final"].shape[1])) + self.assertEqual(out[4].shape, (0, 2)) # globalF (0, channels) + self.assertEqual(out[2].shape, (0,)) # frames + + def test_kept_trial_reads_and_computes(self): + """A kept trial reads (monkeypatched) then computes the traces.""" + inp = _trial_inputs(num_channels=1) + g = inp["g"] + + def fake_load(*args, **kwargs): + """Return the synthetic loaded-trial arrays.""" + return ( + inp["data"], + inp["data2"], + inp["a_data"], + inp["frames"], + ) + + orig = tr._load_high_res_trial_data + tr._load_high_res_trial_data = fake_load + try: + out = tr.get_high_res_traces( + (0, True, inp["background_ds"]), + 0, + 100.0, + np.zeros((g["nsp"], 1), dtype=np.int32), + "unused", + {}, + g["smi"], + g["sh_inds"], + g["sh_vals"], + inp["a_final"].numpy(), # numpy -> exercises tensor conversion + inp["unique_motion_ds"], + inp["mot_keep_ds"], + 0.0, + g["psf2d"], + g["nsp"], + g["nz"], + g["npc"], + g["npr"], + 1, + inp["soma_sps"], + ) + finally: + tr._load_high_res_trial_data = orig + + # matches a direct compute with the same inputs + expected = _compute(inp) + np.testing.assert_allclose( + np.nan_to_num(out[0]), np.nan_to_num(expected[0]), rtol=1e-6 + ) + self.assertEqual(out[0].shape, (inp["n_frames"], 3)) + + +if __name__ == "__main__": + unittest.main() From a709a5aabed8173198d14cf060d446834a7186c9 Mon Sep 17 00:00:00 2001 From: Michael Xie Date: Tue, 28 Jul 2026 10:52:51 -0700 Subject: [PATCH 11/55] implement run code for bandsilo --- README.md | 2 +- pyproject.toml | 4 +- src/giant_python/__init__.py | 4 +- src/giant_python/bandsilo/__init__.py | 4 +- src/giant_python/bandsilo/geometry.py | 2 +- src/giant_python/bandsilo/gui.py | 16 +- src/giant_python/bandsilo/hdf5.py | 2 +- src/giant_python/bandsilo/pipeline.py | 824 ++++++++++++++++++++++++ src/giant_python/bandsilo/traces.py | 4 +- src/giant_python/bandsilo/trial_data.py | 8 +- src/giant_python/gui/draw_rois.py | 1 + src/giant_python/io/__init__.py | 8 +- src/giant_python/io/slap2.py | 8 +- src/giant_python/math/__init__.py | 6 +- src/giant_python/math/peaks.py | 2 +- src/giant_python/models/__init__.py | 7 +- src/giant_python/models/params.py | 7 +- src/giant_python/parallel.py | 12 +- src/giant_python/pipeline/__init__.py | 16 +- src/giant_python/pipeline/base.py | 2 +- src/giant_python/pipeline/extract.py | 71 +- src/giant_python/pipeline/organize.py | 1 + src/giant_python/pipeline/pipeline.py | 1 + src/giant_python/pipeline/register.py | 15 +- tests/test_bandsilo_geometry.py | 2 +- tests/test_bandsilo_peaks.py | 3 +- tests/test_bandsilo_pipeline.py | 238 +++++++ tests/test_bandsilo_trial_data.py | 10 +- tests/test_io.py | 10 +- tests/test_math.py | 10 +- tests/test_package_metadata.py | 6 +- tests/test_parallel.py | 24 +- tests/test_pipeline.py | 116 +++- 33 files changed, 1302 insertions(+), 144 deletions(-) create mode 100644 src/giant_python/bandsilo/pipeline.py create mode 100644 tests/test_bandsilo_pipeline.py diff --git a/README.md b/README.md index f4169dd..b09dc57 100644 --- a/README.md +++ b/README.md @@ -326,7 +326,7 @@ Top-level fields written by **all three** motion correction backends: `numChanne | `yPost` | 1 x 1 | numeric | Maximum row shift searched **after** the reference position (pixels); equals `align_params.maxshiftXY` | | `Path{n}` | — | group | One group per imaging path (DMD) | | `Path{n}/likelihood_means` | Y x X x Z x C x nSP | single | Precomputed expected superpixel mean intensity in the padded reference stack at each displacement in the search cube, per channel and superpixel; used as the template for Poisson or correlation motion inference | -| `Path{n}/allSuperPixelIDs` | nSP x 1 | numeric | Unique superpixel keys for this path: `superPixIdx * 100 + zIdx` (integration-mode pixels only when `integrationOnly` is true) | +| `Path{n}/allSuperPixelIDs` | nSP x 1 | numeric | Unique superpixel keys for this path: `superPixIdx * 100 + zIdx` (band-scan pixels only when `bandOnly` is true) | | `Path{n}/sparseMaskInds` | N x 2 | numeric | Sparse ROI definition: column 1 = linear DMD pixel index (`rows x cols x numFastZs` layout); column 2 = superpixel index (1 … nSP) | | `Path{n}/zPre` | 1 x 1 | numeric | Maximum reference-stack Z shift searched **before** the matched plane (planes); capped by `align_params.maxshiftZ` and available reference Z planes | | `Path{n}/zPost` | 1 x 1 | numeric | Maximum reference-stack Z shift searched **after** the matched plane (planes); capped similarly to `zPre` | diff --git a/pyproject.toml b/pyproject.toml index 7fc2efd..1ade81f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ classifiers = [ readme = "README.md" dynamic = ["version"] -# NOTE: The SLAP2 integration (BandSILo) source-extraction backend requires +# NOTE: The SLAP2 band-scan (BandSILo) source-extraction backend requires # ``slap2_utils`` (raw SLAP2 data-file reader). It is not published on PyPI, so # it is installed directly from source via the git URL below. dependencies = [ @@ -54,7 +54,7 @@ where = ["src"] [tool.setuptools.package-data] # Bundle the default point-spread-function template used by the BandSILo -# (integration) source-extraction backend. +# (band-scan) source-extraction backend. giant_python = ["assets/psfs/*.tif"] [tool.setuptools.dynamic] diff --git a/src/giant_python/__init__.py b/src/giant_python/__init__.py index b613276..48ba9ae 100644 --- a/src/giant_python/__init__.py +++ b/src/giant_python/__init__.py @@ -20,7 +20,7 @@ TrialTable, ) from .pipeline import ( - IntegrationSourceExtractor, + BandSourceExtractor, MotionCorrector, Pipeline, SourceExtractor, @@ -33,8 +33,8 @@ __all__ = [ "AlignmentData", "AlignParams", + "BandSourceExtractor", "ExperimentSummary", - "IntegrationSourceExtractor", "MotionCorrector", "Pipeline", "SiloParams", diff --git a/src/giant_python/bandsilo/__init__.py b/src/giant_python/bandsilo/__init__.py index 6602879..f0f6470 100644 --- a/src/giant_python/bandsilo/__init__.py +++ b/src/giant_python/bandsilo/__init__.py @@ -1,6 +1,6 @@ -"""Self-contained BandSILo (SLAP2 integration/band-scan) backend. +"""Self-contained BandSILo (SLAP2 band-scan) backend. -This package owns everything the integration source-extraction backend needs +This package owns everything the band source-extraction backend needs (HDF5 IO, geometry/PSF, trial-data reading, background/rho, peak detection, NMF localization, trace extraction, and orchestration), ported faithfully from ``extractSLAP2IntegrationSources.py`` in ophys-slap2-analysis. diff --git a/src/giant_python/bandsilo/geometry.py b/src/giant_python/bandsilo/geometry.py index c01c753..89bc087 100644 --- a/src/giant_python/bandsilo/geometry.py +++ b/src/giant_python/bandsilo/geometry.py @@ -165,7 +165,7 @@ def load_reference_stack( ref_stack_meta: dict, dmd_ix: int, ) -> Tuple[Optional[np.ndarray], np.ndarray, Optional[str]]: - """Load the integration reference stack for one DMD. + """Load the band reference stack for one DMD. Reads the REFERENCE tif from ``datadr`` but takes the channel list from the trial table's embedded ref_stack metadata. The raw counts are scaled by diff --git a/src/giant_python/bandsilo/gui.py b/src/giant_python/bandsilo/gui.py index 4ea38fa..a123cdb 100644 --- a/src/giant_python/bandsilo/gui.py +++ b/src/giant_python/bandsilo/gui.py @@ -374,10 +374,10 @@ def _load_one_path( return False out["user_roi_masks"][dmd_key] = roi_masks - out["user_roi_superpixels"][dmd_key] = ( - user_roi_superpixel_lists_from_masks( - roi_masks, geo["sp_fastz"], geo["sp_rows"], geo["sp_cols"] - ) + out["user_roi_superpixels"][ + dmd_key + ] = user_roi_superpixel_lists_from_masks( + roi_masks, geo["sp_fastz"], geo["sp_rows"], geo["sp_cols"] ) out["user_roi_labels"][dmd_key] = labels out["roi_records"][dmd_key] = recs @@ -541,10 +541,10 @@ def run_user_roi_selection( g = user_roi_geo[dmd_key] roi_masks, roi_records = _select_rois_for_dmd(cv2, dmd_ix, g) out["user_roi_masks"][dmd_key] = roi_masks - out["user_roi_superpixels"][dmd_key] = ( - user_roi_superpixel_lists_from_masks( - roi_masks, g["sp_fastz"], g["sp_rows"], g["sp_cols"] - ) + out["user_roi_superpixels"][ + dmd_key + ] = user_roi_superpixel_lists_from_masks( + roi_masks, g["sp_fastz"], g["sp_rows"], g["sp_cols"] ) out["user_roi_labels"][dmd_key] = [r["label"] for r in roi_records] out["roi_records"][dmd_key] = roi_records diff --git a/src/giant_python/bandsilo/hdf5.py b/src/giant_python/bandsilo/hdf5.py index f6610b4..bcbcb45 100644 --- a/src/giant_python/bandsilo/hdf5.py +++ b/src/giant_python/bandsilo/hdf5.py @@ -268,7 +268,7 @@ def load_trial_table(path: Union[str, Path]) -> dict: """Load and normalize a GIAnT ``trial_table.h5`` for BandSILo. Reads the trial table written by BandRegistration.m and assembles the - normalized structure consumed by the integration backend (mirrors the + normalized structure consumed by the band backend (mirrors the setup block of ``main`` in ``extractSLAP2IntegrationSources.py``). Parameters diff --git a/src/giant_python/bandsilo/pipeline.py b/src/giant_python/bandsilo/pipeline.py new file mode 100644 index 0000000..ce4f270 --- /dev/null +++ b/src/giant_python/bandsilo/pipeline.py @@ -0,0 +1,824 @@ +"""BandSILo driver: compose Phases 1-7 and write experiment_summary.h5. + +Ported from ``main()`` in ``extractSLAP2IntegrationSources.py`` (ref +1360-3108). :func:`extract_band_sources` runs the whole band-scan +source-extraction pipeline for one ``trial_table.h5``: load inputs +and geometry, read low-res trial data, estimate background/noise/rho, build the +activity image and detect peaks, localize sources with NMF, extract high-res +traces, assemble dF/F, and write the byte-compatible ``experiment_summary.h5`` +before returning a populated :class:`~giant_python.models.ExperimentSummary`. + +The orchestration driver itself touches ``slap2_utils`` reads, torch NMF, and +disk IO end-to-end, so it cannot be exercised without a full dataset and is +marked ``# pragma: no cover`` (its constituent Phase 1-7 kernels are each +unit-tested and cross-checked against the reference). The output assembly and +HDF5 writer are pure and fully tested here, since the on-disk schema is a +frozen contract. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import List, Optional, Union + +import h5py +import numpy as np +import torch + +from ..models.experiment import ( + ExperimentSummary, + Source, + UserRoi, + Visualizations, +) +from ..models.params import SiloParams +from ..parallel import map_trials +from . import background as bg +from . import geometry as geo +from . import summary_images as si +from . import trial_data as td +from .baseline import assemble_dff +from .hdf5 import load_trial_table, to_serializable, write_dict_to_h5group +from .nmf import fit_sources +from .peaks import get_act_im_peaks +from .traces import get_high_res_traces + +_FRAME_INFO_KEYS = ( + "trial_num_frames", + "discard_frames", + "frame_line_idxs", + "offlineXshifts", + "offlineYshifts", + "offlineZshifts", + "onlineXshifts", + "onlineYshifts", + "onlineZshifts", +) + + +@dataclass +class PathResult: + """Per-DMD (``Path{N}``) results feeding the output writer. + + Bundles the localized sources, per-trial trace results, and summary images + for one imaging path, plus the geometry and windows needed to assemble the + ``experiment_summary.h5`` datasets. + + Attributes + ---------- + a : torch.Tensor + Spatial profiles ``(n_pixels, n_sources)``. + source_params : torch.Tensor + Source parameters ``(n_sources, 6)``. + source_snr : ndarray + Per-source low-resolution SNR ``(n_sources,)``. + act_im, mean_im, act_im_peaks : ndarray + Activity image, mean image, and raw activity-image peaks. + trace_results : list + Per-trial 8-tuples from + :func:`giant_python.bandsilo.traces.get_high_res_traces`. + z_depths : ndarray + Per-fast-Z reference-stack depth (``Path{N}/Z_depths``). + num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row, num_channels : int + Geometry / channel count. + denoise_window, baseline_window : int + Windows passed to :func:`giant_python.bandsilo.baseline.compute_f0`. + draw_user_rois : bool + Whether user ROIs were drawn (gates writing the user-ROI group). + soma_masks : list or None + Per-ROI boolean masks (``(fastz, rows, cols)`` each). + soma_labels : list or None + Per-ROI text labels. + yx_shape : tuple + ``(rows, cols)`` plane shape (for the empty-ROI mask shape). + """ + + a: torch.Tensor + source_params: torch.Tensor + source_snr: np.ndarray + act_im: np.ndarray + mean_im: np.ndarray + act_im_peaks: np.ndarray + trace_results: list + z_depths: np.ndarray + num_fast_zs: int + dmd_pixels_per_column: int + dmd_pixels_per_row: int + num_channels: int + denoise_window: int + baseline_window: int + draw_user_rois: bool = False + soma_masks: Optional[list] = None + soma_labels: Optional[list] = None + yx_shape: tuple = (0, 0) + + +def _with_nan_channel(arr: np.ndarray, num_channels: int) -> np.ndarray: + """Reshape ``(frames, sources)`` to ``(sources, channels, frames)``. + + Only the activity channel (index 0) is filled; the remaining channels are + NaN (reserved until multi-channel source extraction exists). + + Parameters + ---------- + arr : ndarray of shape (n_frames, n_sources) + Per-source trace. + num_channels : int + Number of acquisition channels. + + Returns + ------- + ndarray of shape (n_sources, num_channels, n_frames) + The reshaped trace with NaN in the non-activity channels. + """ + out = np.full( + (arr.shape[1], num_channels, arr.shape[0]), np.nan, dtype=arr.dtype + ) + out[:, 0, :] = arr.T + return out + + +def _assemble_user_rois(pr: PathResult) -> Optional[dict]: + """Assemble the ``user_rois`` datasets for one path, or ``None``. + + Parameters + ---------- + pr : PathResult + The path's results (uses ``draw_user_rois``/``soma_masks``/ + ``soma_labels`` and the per-trial ``F_soma`` from ``trace_results``). + + Returns + ------- + dict or None + ``{"mask", "labels", "F"}`` when user ROIs were drawn, else ``None``. + """ + if not pr.draw_user_rois or pr.soma_masks is None: + return None + if len(pr.soma_masks) > 0: + masks_by_roi = np.stack( + [np.asarray(m, dtype=bool) for m in pr.soma_masks], axis=0 + ) + else: + masks_by_roi = np.zeros((0, pr.num_fast_zs, *pr.yx_shape), dtype=bool) + f_soma_all = np.concatenate( + [r[7] for r in pr.trace_results], axis=0 + ).transpose(1, 2, 0) + return {"mask": masks_by_roi, "labels": pr.soma_labels, "F": f_soma_all} + + +def assemble_path_outputs(pr: PathResult) -> dict: + """Assemble one path's HDF5-ready datasets from its results. + + Concatenates the per-trial traces, re-estimates ``F0`` and forms the + per-source dF/F, and shapes every ``Path{N}`` dataset (sources, frame info, + global fluorescence, visualizations, optional user ROIs). + + Parameters + ---------- + pr : PathResult + The path's results. + + Returns + ------- + dict + Keys mirror the ``experiment_summary.h5`` ``Path{N}`` datasets: + ``z_depths``, ``profiles``, ``coords``, ``dF_ls``, ``F0``, ``SNR``, + the frame-info arrays, ``global_f``, ``act_im``, ``mean_im``, + ``act_im_peaks``, and ``user_rois`` (or ``None``). + """ + results = pr.trace_results + d_f = np.concatenate([r[0] for r in results], axis=0) + f0_ls = np.concatenate([r[1] for r in results], axis=0) + f_full, f0, d_f, _ = assemble_dff( + d_f, f0_ls, pr.denoise_window, pr.baseline_window + ) + + profiles = pr.a.numpy().T.reshape( + -1, pr.num_fast_zs, pr.dmd_pixels_per_column, pr.dmd_pixels_per_row + ) + + return { + "z_depths": pr.z_depths, + "profiles": profiles, + "coords": pr.source_params[:, :3].numpy(), + "dF_ls": _with_nan_channel(d_f, pr.num_channels), + "F0": _with_nan_channel(f0, pr.num_channels), + "SNR": pr.source_snr.reshape(-1, 1), + "trial_num_frames": np.concatenate( + [[len(r[2])] for r in results] + ).reshape(-1, 1), + "discard_frames": np.any(np.isnan(f_full), axis=1).reshape(-1, 1), + "frame_line_idxs": np.concatenate( + [r[2] for r in results], axis=0 + ).reshape(-1, 1), + "offlineXshifts": np.concatenate( + [r[5][1] for r in results], axis=0 + ).reshape(-1, 1), + "offlineYshifts": np.concatenate( + [r[5][0] for r in results], axis=0 + ).reshape(-1, 1), + "offlineZshifts": np.concatenate( + [r[5][2] for r in results], axis=0 + ).reshape(-1, 1), + "onlineXshifts": np.concatenate( + [r[6][1] for r in results], axis=0 + ).reshape(-1, 1), + "onlineYshifts": np.concatenate( + [r[6][0] for r in results], axis=0 + ).reshape(-1, 1), + "onlineZshifts": np.concatenate( + [r[6][2] for r in results], axis=0 + ).reshape(-1, 1), + "global_f": np.concatenate([r[4] for r in results], axis=0).T, + "act_im": pr.act_im, + "mean_im": pr.mean_im, + "act_im_peaks": pr.act_im_peaks, + "user_rois": _assemble_user_rois(pr), + } + + +def _write_path_group(f: h5py.File, dmd_ix: int, assembled: dict) -> None: + """Write one ``Path{N}`` group from assembled outputs (overwriting).""" + group_name = f"Path{dmd_ix + 1}" + if group_name in f: + del f[group_name] + dmd_group = f.create_group(group_name) + dmd_group.create_dataset("Z_depths", data=assembled["z_depths"]) + + sources = dmd_group.create_group("sources") + spatial = sources.create_group("spatial") + spatial.create_dataset("profiles", data=assembled["profiles"]) + spatial.create_dataset("coords", data=assembled["coords"]) + temporal = sources.create_group("temporal") + temporal.create_dataset("dF_ls", data=assembled["dF_ls"]) + temporal.create_dataset("F0", data=assembled["F0"]) + temporal.create_dataset("SNR", data=assembled["SNR"]) + + frame_group = dmd_group.create_group("frame_info") + for key in _FRAME_INFO_KEYS: + frame_group.create_dataset(key, data=assembled[key]) + + global_group = dmd_group.create_group("global") + global_group.create_dataset("F", data=assembled["global_f"]) + + vis = dmd_group.create_group("visualizations") + vis.create_dataset("act_im", data=assembled["act_im"]) + vis.create_dataset("mean_im", data=assembled["mean_im"]) + vis.create_dataset("act_im_peaks", data=assembled["act_im_peaks"]) + + if assembled["user_rois"] is not None: + rois = assembled["user_rois"] + user_roi_group = dmd_group.create_group("user_rois") + user_roi_group.create_dataset("mask", data=rois["mask"]) + user_roi_group.create_dataset( + "labels", + data=np.array(rois["labels"], dtype=object).reshape(-1, 1), + dtype=h5py.string_dtype(encoding="utf-8"), + ) + user_roi_group.create_dataset("F", data=rois["F"]) + + +def write_experiment_summary( + output_path: Union[str, Path], + params_serializable: dict, + assembled_paths: List[dict], +) -> None: + """Write the byte-compatible ``experiment_summary.h5``. + + Writes the root ``row_major`` flag and ``params`` group once, then one + ``Path{N}`` group per DMD (overwriting any existing group of that name). + + Parameters + ---------- + output_path : str or Path + Destination ``experiment_summary.h5`` path. + params_serializable : dict + The analysis parameters (already run through + :func:`giant_python.bandsilo.hdf5.to_serializable`). + assembled_paths : list of dict + Per-path outputs from :func:`assemble_path_outputs`. + """ + with h5py.File(output_path, "a") as f: + if "row_major" not in f: + f["row_major"] = 1 + if "params" not in f: + write_dict_to_h5group( + f.create_group("params"), params_serializable + ) + for dmd_ix, assembled in enumerate(assembled_paths): + _write_path_group(f, dmd_ix, assembled) + + +def _sources_from_assembled(assembled: dict) -> List[Source]: + """Build the per-source :class:`Source` list for one path.""" + profiles = assembled["profiles"] + coords = assembled["coords"] + d_f = assembled["dF_ls"] + f0 = assembled["F0"] + snr = assembled["SNR"] + return [ + Source( + profile=profiles[j], + coords=coords[j], + df_ls=d_f[j], + f0=f0[j], + snr=float(snr[j, 0]), + ) + for j in range(profiles.shape[0]) + ] + + +def _user_rois_from_assembled(assembled: dict) -> List[UserRoi]: + """Build the per-ROI :class:`UserRoi` list for one path.""" + rois = assembled["user_rois"] + if rois is None: + return [] + labels = rois["labels"] + masks = rois["mask"] + f_soma = rois["F"] + return [ + UserRoi( + label=str(labels[k]), + mask=masks[k] if k < masks.shape[0] else None, + f=f_soma[k] if k < f_soma.shape[0] else None, + ) + for k in range(len(labels)) + ] + + +def build_experiment_summary( + params_serializable: dict, assembled_paths: List[dict] +) -> ExperimentSummary: + """Build the returned :class:`ExperimentSummary` from assembled outputs. + + Populates per-path sources and user ROIs; the single-view visualization / + z-depth / global-F / frame-info fields hold the last path's values (the + per-path structure lives in ``experiment_summary.h5``). + + Parameters + ---------- + params_serializable : dict + The analysis parameters. + assembled_paths : list of dict + Per-path outputs from :func:`assemble_path_outputs`. + + Returns + ------- + ExperimentSummary + The populated summary. + """ + summary = ExperimentSummary(params=params_serializable) + for assembled in assembled_paths: + summary.sources.append(_sources_from_assembled(assembled)) + summary.user_rois.append(_user_rois_from_assembled(assembled)) + summary.visualizations = Visualizations( + mean_im=assembled["mean_im"], + act_im=assembled["act_im"], + act_im_peaks=assembled["act_im_peaks"], + ) + summary.z_depths = assembled["z_depths"] + summary.global_f = assembled["global_f"] + summary.frame_info = {k: assembled[k] for k in _FRAME_INFO_KEYS} + return summary + + +def _resolve_params( + params_in, +) -> SiloParams: # pragma: no cover - trivial glue + """Coerce ``params_in`` (None / dict / SiloParams) to a SiloParams.""" + if params_in is None: + return SiloParams(scan_mode="band") + if isinstance(params_in, SiloParams): + return params_in + return SiloParams(**params_in) + + +def extract_band_sources( + path_to_trial_table: Union[str, Path], + params_in=None, +) -> ExperimentSummary: # pragma: no cover - full IO/slap2/torch orchestration + """Run the band source-extraction pipeline for one trial table. + + Loads the ``trial_table.h5`` and its geometry, then for each DMD path runs + the low-res read, background/rho estimation, activity-image peak detection, + NMF localization, and high-res trace extraction, writes + ``experiment_summary.h5``, and returns a populated + :class:`~giant_python.models.ExperimentSummary`. + + This driver reads SLAP2 files (``slap2_utils``), runs the torch NMF, and + performs disk IO, so it is not unit-tested (its Phase 1-7 kernels are). + + Parameters + ---------- + path_to_trial_table : str or Path + Path to the ``trial_table.h5`` produced by BandRegistration. + params_in : SiloParams or dict, optional + Parameter overrides. + + Returns + ------- + ExperimentSummary + The extracted sources and summary (also written to disk). + """ + params = _resolve_params(params_in) + trial_table = load_trial_table(path_to_trial_table) + result_dr = str(trial_table["savedr"]) + src_extr_dr = Path(result_dr) / "source_extraction" + src_extr_dr.mkdir(parents=True, exist_ok=True) + output_path = src_extr_dr / "experiment_summary.h5" + + lookup = geo.load_lookup_table( + Path(result_dr) / "motion_correction" / "bandRegLookupTable.h5", + trial_table["n_dmds"], + ) + psf = geo.load_psf(params.psf_dilation, trial_table["n_dmds"]) + + assembled_paths = [] + for dmd_ix in range(trial_table["n_dmds"]): + assembled_paths.append( + assemble_path_outputs( + _process_dmd(dmd_ix, trial_table, params, lookup, psf) + ) + ) + + params_serializable = to_serializable(_params_dict(params)) + write_experiment_summary(output_path, params_serializable, assembled_paths) + return build_experiment_summary(params_serializable, assembled_paths) + + +def _params_dict(params: SiloParams) -> dict: # pragma: no cover - glue + """Return the on-disk params dict written to ``experiment_summary.h5``.""" + return { + "numChannels": params.num_channels, + "analyzeHz": params.analyze_hz, + "denoiseWindow_s": params.denoise_window_s, + "baselineWindow_s": params.baseline_window_s, + "decayTau_s": params.decay_tau_s, + "dXY": params.d_xy, + "sparse_fac": params.sparse_fac, + "vif": params.vif, + "peakth": params.peakth, + "peak_buffer": params.peak_buffer, + "draw_user_rois": params.draw_user_rois, + "operator": params.operator, + } + + +def _process_dmd( + dmd_ix: int, + trial_table: dict, + params: SiloParams, + lookup: dict, + psf: dict, +) -> PathResult: # pragma: no cover - full per-DMD IO/slap2/torch compute + """Run Phases 3-7 for one DMD path and bundle the results. + + Reads the low-res trial data, estimates background/noise/rho, builds the + activity image and detects peaks, localizes sources with NMF, and extracts + high-res traces per trial (fanned out via + :func:`giant_python.parallel.map_trials`). + """ + key = f"DMD{dmd_ix + 1}" + datadr = str(trial_table["datadr"]) + n_trials = trial_table["filename"].shape[1] + num_channels = params.num_channels + align_hz = trial_table["align_hz"][key] + keep_trials = trial_table["keep_trials"] + + ref_stack, _, _ = geo.load_reference_stack( + datadr, trial_table["ref_stack"], dmd_ix + ) + dmd_pixels_per_column = ref_stack.shape[2] + dmd_pixels_per_row = ref_stack.shape[3] + yx_shape = (dmd_pixels_per_column, dmd_pixels_per_row) + fastz_to_refz = lookup["fastZ2RefZ"][key] + num_fast_zs = fastz_to_refz.shape[0] + all_sp_ids = lookup["allSuperPixelIDs"][key] + num_super_pixels = all_sp_ids.shape[0] + n_pixels = num_fast_zs * dmd_pixels_per_column * dmd_pixels_per_row + + smi = geo.build_subsample_matrix_inds( + all_sp_ids, lookup["sparseMaskInds"][key] + ) + psf2d = psf[key] + sparse_h_inds, sparse_h_vals = geo.build_sparse_h( + smi, psf2d, dmd_pixels_per_column, dmd_pixels_per_row + ) + ref_d, ref_c, ref_r = geo.ref_pixs_to_drc( + smi[:, 0], dmd_pixels_per_column, dmd_pixels_per_row + ) + + def _read(trial_ix): + """Read one trial's low-res data (per-trial worker).""" + return td.read_band_trial_data( + trial_ix, + bool(keep_trials[dmd_ix, trial_ix]), + dmd_ix, + align_hz, + all_sp_ids, + datadr, + trial_table, + num_channels, + ) + + low = td.assemble_lowres_data( + map_trials(_read, range(n_trials), params.max_workers), num_channels + ) + low_res_data_norm = low["lowResData"] / low["lowResDataCt"] + v_im = 1.0 / low["lowResDataCt"] + low2 = ( + low["lowResData2"] / low["lowResDataCt2"] + if num_channels >= 2 + else None + ) + + unique_motion, mot_inds = bg.bin_motion( + low["lowResMotionR"], low["lowResMotionC"], low["lowResMotionZ"] + ) + median_z = np.median(low["lowResMotionZ"]) + _, frames_to_keep = bg.select_motion_bins( + unique_motion, mot_inds, low["lowResMotionZ"] + ) + mean_im = si.compute_mean_image( + low_res_data_norm, + unique_motion, + mot_inds, + frames_to_keep, + ref_d, + ref_r, + ref_c, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + num_channels, + low2, + ) + umyx, mot_inds_yx = bg.bin_motion_yx( + low["lowResMotionR"], low["lowResMotionC"], frames_to_keep + ) + _, sel_pix_idxs = bg.build_selected_pixel_mask( + umyx, + ref_d, + ref_r, + ref_c, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + psf2d, + ) + pixel_coords = bg.pixel_coords_from_idxs( + sel_pix_idxs, dmd_pixels_per_column, dmd_pixels_per_row + ) + + background = _estimate_background( + low_res_data_norm, + umyx, + mot_inds_yx, + sel_pix_idxs, + ref_d, + ref_r, + ref_c, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + psf2d, + num_super_pixels, + align_hz, + params, + ) + data_std, _, _ = bg.fit_noise_variance_model( + low_res_data_norm, background, v_im, params.vif + ) + residual = bg.compute_residual(low_res_data_norm, background, data_std) + + act_im, act_im_peaks, a, source_params, source_snr = _localize( + residual, + umyx, + mot_inds_yx, + sel_pix_idxs, + pixel_coords, + n_pixels, + ref_d, + ref_r, + ref_c, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + psf2d, + num_super_pixels, + sparse_h_inds, + sparse_h_vals, + align_hz, + params, + ) + + def _traces(trial_ix): + """Extract one trial's high-res traces (per-trial worker).""" + return get_high_res_traces( + ( + trial_ix, + bool(keep_trials[dmd_ix, trial_ix]), + background[:, low["lowResTrialID"] == trial_ix], + ), + dmd_ix, + params.analyze_hz, + all_sp_ids, + datadr, + trial_table, + smi, + sparse_h_inds, + sparse_h_vals, + a, + umyx, + np.arange(umyx.shape[0]), + median_z, + psf2d, + num_super_pixels, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + num_channels, + [], + ) + + trace_results = map_trials(_traces, range(n_trials), params.max_workers) + trace_results = [r for r in trace_results if r[0].shape[0] > 0] + + return PathResult( + a=a, + source_params=source_params, + source_snr=source_snr, + act_im=act_im, + mean_im=mean_im, + act_im_peaks=act_im_peaks, + trace_results=trace_results, + z_depths=fastz_to_refz, + num_fast_zs=num_fast_zs, + dmd_pixels_per_column=dmd_pixels_per_column, + dmd_pixels_per_row=dmd_pixels_per_row, + num_channels=num_channels, + denoise_window=int( + np.ceil(params.denoise_window_s * params.analyze_hz) + ), + baseline_window=int( + np.ceil(params.baseline_window_s * params.analyze_hz) + ), + draw_user_rois=params.draw_user_rois, + yx_shape=yx_shape, + ) + + +def _estimate_background( + low_res_data_norm, + umyx, + mot_inds_yx, + sel_pix_idxs, + ref_d, + ref_r, + ref_c, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + psf2d, + num_super_pixels, + align_hz, + params, +): # pragma: no cover - Phase-4 background composition over real data + """Interpolated + rolling-baseline background on the superpixel grid.""" + psf_tensor, psf_center, psf_exp, psf_center_exp = bg.expand_psf(psf2d) + interp_data = np.full( + (sel_pix_idxs.shape[0], low_res_data_norm.shape[1]), + np.nan, + dtype=np.float32, + ) + for z in range(num_fast_zs): + z_idxs, sel_2d = bg.selected_pixels_2d_for_plane( + sel_pix_idxs, z, dmd_pixels_per_column, dmd_pixels_per_row + ) + if z_idxs.size == 0: + continue + ref_z = np.flatnonzero(ref_d == z) + interp_data[z_idxs], _, _ = bg.build_interp_data( + low_res_data_norm[ref_z], + ref_r[ref_z], + ref_c[ref_z], + sel_2d, + umyx, + mot_inds_yx, + ) + baseline_window = bg.baseline_window_frames( + align_hz, params.baseline_window_s + ) + interp_background = bg.compute_rolling_baseline( + interp_data, baseline_window + ) + return bg.assemble_background( + interp_background, + umyx, + mot_inds_yx, + sel_pix_idxs, + ref_d, + ref_r, + ref_c, + low_res_data_norm.shape[0], + low_res_data_norm.shape[1], + dmd_pixels_per_column, + dmd_pixels_per_row, + ) + + +def _localize( + residual, + umyx, + mot_inds_yx, + sel_pix_idxs, + pixel_coords, + n_pixels, + ref_d, + ref_r, + ref_c, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + psf2d, + num_super_pixels, + sparse_h_inds, + sparse_h_vals, + align_hz, + params, +): # pragma: no cover - Phase 4-6 rho/activity/NMF over real data + """Compute rho + activity image, detect peaks, and NMF-fit sources.""" + h_mots = bg.build_motion_h_matrices( + sparse_h_inds, + sparse_h_vals, + umyx, + sel_pix_idxs, + num_super_pixels, + dmd_pixels_per_row, + ) + psf_tensor, psf_center, psf_exp, psf_center_exp = bg.expand_psf(psf2d) + d_mats, d_mats_exp = [], [] + for z in range(num_fast_zs): + _, sel_2d = bg.selected_pixels_2d_for_plane( + sel_pix_idxs, z, dmd_pixels_per_column, dmd_pixels_per_row + ) + d_mats.append( + bg.build_convolution_matrix(sel_2d, psf_tensor, psf_center) + ) + d_mats_exp.append( + bg.build_convolution_matrix(sel_2d, psf_exp, psf_center_exp) + ) + rho = bg.compute_rho( + residual, + mot_inds_yx, + umyx, + h_mots, + d_mats, + d_mats_exp, + sel_pix_idxs, + ref_d, + ref_r, + ref_c, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + psf2d, + ) + nan_ct = bg.mask_high_nan_rho(rho) + rho = bg.smooth_rho(rho, bg.decay_kernel_1d(params.decay_tau_s, align_hz)) + + act_im = si.accumulate_activity_image( + rho, + sel_pix_idxs, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + ) + act_im = si.finalize_activity_image(act_im, sel_pix_idxs, nan_ct) + + peak_th = params.peakth if params.peakth is not None else 8 + source_seeds = get_act_im_peaks( + act_im, peak_th=peak_th, buffer_size=params.peak_buffer + ) + act_im_peaks = source_seeds.copy() + + result = fit_sources( + source_seeds, + residual, + h_mots, + umyx, + mot_inds_yx, + sel_pix_idxs, + pixel_coords, + n_pixels, + params.d_xy, + params.sparse_fac, + ) + return ( + act_im, + act_im_peaks, + result["A"], + result["source_params"], + result["source_snr"], + ) diff --git a/src/giant_python/bandsilo/traces.py b/src/giant_python/bandsilo/traces.py index ceaefb6..7ca9aaf 100644 --- a/src/giant_python/bandsilo/traces.py +++ b/src/giant_python/bandsilo/traces.py @@ -26,7 +26,7 @@ from .background import build_motion_h_matrices, build_selected_pixel_mask from .geometry import ref_pixs_to_drc from .nmf import solve_phi_motion -from .trial_data import nearest_interp, read_integration_trial_data +from .trial_data import nearest_interp, read_band_trial_data def _empty_trace_result( @@ -486,7 +486,7 @@ def _load_high_res_trial_data( tuple ``(data, data2, a_data, frames)`` with count-normalized activity. """ - result = read_integration_trial_data( + result = read_band_trial_data( trial_ix, True, dmd_ix, diff --git a/src/giant_python/bandsilo/trial_data.py b/src/giant_python/bandsilo/trial_data.py index 8910cb1..8e67a93 100644 --- a/src/giant_python/bandsilo/trial_data.py +++ b/src/giant_python/bandsilo/trial_data.py @@ -1,4 +1,4 @@ -"""Read low-resolution superpixel activity for integration (band-scan) trials. +"""Read low-resolution superpixel activity for band-scan trials. Ported from ``extractSLAP2IntegrationSources.py`` (``get_trial_data`` ref 715-842, ``nearest_interp``/``fast_dilation`` ref 166-205, and the per-trial @@ -291,7 +291,7 @@ def _open_slap2_file(path: str): # pragma: no cover - requires slap2_utils return slap2_utils.DataFile(path) -def read_integration_trial_data( +def read_band_trial_data( trial_ix: int, keep_trial: bool, dmd_ix: int, @@ -302,7 +302,7 @@ def read_integration_trial_data( num_channels: int, all_channels: bool = True, ) -> Optional[dict]: - """Read one integration trial's low-res superpixel activity. + """Read one band trial's low-res superpixel activity. Opens the trial's SLAP2 file, builds the downsampled-frame grid, accumulates weighted superpixel activity, and loads the matching alignment @@ -376,7 +376,7 @@ def assemble_lowres_data(results: list, num_channels: int) -> dict: Parameters ---------- results : list - Per-trial dicts from :func:`read_integration_trial_data` (or ``None``). + Per-trial dicts from :func:`read_band_trial_data` (or ``None``). num_channels : int Number of acquisition channels (>= 2 adds second-channel arrays). diff --git a/src/giant_python/gui/draw_rois.py b/src/giant_python/gui/draw_rois.py index 2c55e91..c90239a 100644 --- a/src/giant_python/gui/draw_rois.py +++ b/src/giant_python/gui/draw_rois.py @@ -23,6 +23,7 @@ class DrawROIs: """ def __init__(self, trial_table: "object") -> None: + """Store the trial table providing image paths and the save dir.""" self.trial_table = trial_table def show(self) -> None: diff --git a/src/giant_python/io/__init__.py b/src/giant_python/io/__init__.py index 539b30d..742a87f 100644 --- a/src/giant_python/io/__init__.py +++ b/src/giant_python/io/__init__.py @@ -7,17 +7,13 @@ """ from .hdf5 import load_struct_h5, save_struct_h5 -from .slap2 import ( - get_online_motion, - read_integration_trial_data, - ref_pixs_to_drc, -) +from .slap2 import get_online_motion, read_band_trial_data, ref_pixs_to_drc from .tiff import scanimagetiff_data_wrapper, scanimagetiff_wrapper __all__ = [ "get_online_motion", "load_struct_h5", - "read_integration_trial_data", + "read_band_trial_data", "ref_pixs_to_drc", "save_struct_h5", "scanimagetiff_data_wrapper", diff --git a/src/giant_python/io/slap2.py b/src/giant_python/io/slap2.py index b762d2f..b69e810 100644 --- a/src/giant_python/io/slap2.py +++ b/src/giant_python/io/slap2.py @@ -2,8 +2,8 @@ Intended home for the SLAP2 data-file reader (equivalent of MATLAB's ``slap2.Slap2DataFile``) plus the online motion-offset retrieval ported from -getOnlineMotion.m. Also hosts the integration (band-scan) superpixel readers -used by the integration source-extraction backend. +getOnlineMotion.m. Also hosts the band-scan superpixel readers +used by the band source-extraction backend. """ from typing import Optional, Tuple @@ -35,13 +35,13 @@ def ref_pixs_to_drc( raise NotImplementedError -def read_integration_trial_data( +def read_band_trial_data( data_file: object, ds_frames: np.ndarray, super_pixel_ids: np.ndarray, activity_channel: Optional[int] = None, ) -> Tuple[np.ndarray, np.ndarray]: - """Read superpixel-binned activity for one integration (band-scan) trial. + """Read superpixel-binned activity for one band-scan trial. Accumulates weighted line data into per-superpixel time series over the downsampled frame grid. Port of ``get_trial_data`` from diff --git a/src/giant_python/math/__init__.py b/src/giant_python/math/__init__.py index e0d7c6a..987d791 100644 --- a/src/giant_python/math/__init__.py +++ b/src/giant_python/math/__init__.py @@ -7,11 +7,7 @@ from .baseline import compute_f0 from .interpolation import interp_frame -from .peaks import ( - detect_peaks_2d, - gaussian_peaks_integrated, - get_act_im_peaks, -) +from .peaks import detect_peaks_2d, gaussian_peaks_integrated, get_act_im_peaks from .registration import ( dft_register_clipped, xcorr2_nans, diff --git a/src/giant_python/math/peaks.py b/src/giant_python/math/peaks.py index dd357d4..34333da 100644 --- a/src/giant_python/math/peaks.py +++ b/src/giant_python/math/peaks.py @@ -1,6 +1,6 @@ """Activity-image peak detection via integrated-Gaussian fitting. -Shared by both source-extraction backends (standard and integration). Ports +Shared by both source-extraction backends (standard and band). Ports ``getActImPeaks.m`` (GIAnT-MATLAB) / ``get_act_im_peaks`` and its integrated-Gaussian helpers from ``extractSLAP2IntegrationSources.py``. """ diff --git a/src/giant_python/models/__init__.py b/src/giant_python/models/__init__.py index 3257a91..d2e802d 100644 --- a/src/giant_python/models/__init__.py +++ b/src/giant_python/models/__init__.py @@ -7,12 +7,7 @@ """ from .alignment import AlignmentData -from .experiment import ( - ExperimentSummary, - Source, - UserRoi, - Visualizations, -) +from .experiment import ExperimentSummary, Source, UserRoi, Visualizations from .params import AlignParams, SiloParams, set_params from .trial_table import Slap2Info, TrialTable diff --git a/src/giant_python/models/params.py b/src/giant_python/models/params.py index f0a691d..0b4f849 100644 --- a/src/giant_python/models/params.py +++ b/src/giant_python/models/params.py @@ -54,7 +54,7 @@ class SiloParams: ``"slap2"`` or ``"bergamo"``. scan_mode : str SLAP2 scan mode selecting the extraction backend: - ``"standard"`` (pixel-movie SILo) or ``"integration"`` (the + ``"standard"`` (pixel-movie SILo) or ``"band"`` (the band-scan / superpixel variant, paired with BandRegistration). sigma_px : float Spatial smoothing sigma, in pixels. @@ -72,14 +72,15 @@ class SiloParams: activity_channel : int Channel index used for activity detection. draw_user_rois : bool - Whether to prompt for user ROIs before extraction. + Whether to prompt for / use user ROIs; also gates writing the + ``user_rois`` group and per-ROI traces to ``experiment_summary.h5``. analyze_hz : float or None Temporal analysis rate, in Hz. When ``None``, the BandSILo backend falls back to ``100``. Notes ----- - The remaining attributes below are specific to the ``"integration"`` + The remaining attributes below are specific to the ``"band"`` (BandSILo) backend and mirror the GUI defaults in ``extractSLAP2IntegrationSources.py``. They are ignored by the standard (pixel-movie) SILo backend. diff --git a/src/giant_python/parallel.py b/src/giant_python/parallel.py index a6a102f..3bdff5c 100644 --- a/src/giant_python/parallel.py +++ b/src/giant_python/parallel.py @@ -18,6 +18,10 @@ def map_trials( ) -> List[R]: """Apply *func* to each trial, optionally in parallel. + Runs serially when ``n_workers <= 1`` (or a single trial), otherwise fans + the trials out over a joblib process pool. Results are returned in input + order regardless of backend. + Parameters ---------- func : callable @@ -32,4 +36,10 @@ def map_trials( list Results in input order. """ - raise NotImplementedError + trials = list(trials) + if n_workers <= 1 or len(trials) <= 1: + return [func(t) for t in trials] + + from joblib import Parallel, delayed + + return Parallel(n_jobs=n_workers)(delayed(func)(t) for t in trials) diff --git a/src/giant_python/pipeline/__init__.py b/src/giant_python/pipeline/__init__.py index 110b418..7bebdbc 100644 --- a/src/giant_python/pipeline/__init__.py +++ b/src/giant_python/pipeline/__init__.py @@ -8,10 +8,10 @@ from .base import Stage from .extract import ( - IntegrationSourceExtractor, + BandSourceExtractor, SourceExtractor, StandardSourceExtractor, - extract_integration_sources, + extract_band_sources, silo, ) from .organize import ( @@ -22,22 +22,22 @@ ) from .pipeline import Pipeline from .register import ( - IntegrationRegistration, + BandRegistration, MotionCorrector, MultiRoiRegistration, StripRegistration, - integration_registration, + band_registration, multi_roi_registration, strip_registration, ) __all__ = [ + "band_registration", + "BandRegistration", + "BandSourceExtractor", "build_trial_table", "build_trial_table_slap2", - "extract_integration_sources", - "integration_registration", - "IntegrationRegistration", - "IntegrationSourceExtractor", + "extract_band_sources", "MotionCorrector", "MultiRoiRegistration", "multi_roi_registration", diff --git a/src/giant_python/pipeline/base.py b/src/giant_python/pipeline/base.py index a61589d..dffb4ba 100644 --- a/src/giant_python/pipeline/base.py +++ b/src/giant_python/pipeline/base.py @@ -20,4 +20,4 @@ class Stage(ABC): @abstractmethod def run(self, *args: Any, **kwargs: Any) -> Any: """Execute the stage and return its result.""" - raise NotImplementedError + raise NotImplementedError # pragma: no cover - abstract stub diff --git a/src/giant_python/pipeline/extract.py b/src/giant_python/pipeline/extract.py index 34a1d7a..f94b8d4 100644 --- a/src/giant_python/pipeline/extract.py +++ b/src/giant_python/pipeline/extract.py @@ -2,7 +2,7 @@ Exposes the porting-faithful extraction functions plus a :class:`SourceExtractor` base with one backend per SLAP2 scan mode -(standard pixel-movie SILo / integration band-scan SILo), selected via +(standard pixel-movie SILo / band-scan SILo), selected via :meth:`SourceExtractor.for_scan_mode` -- mirroring the :class:`~giant_python.pipeline.register.MotionCorrector` backend pattern. @@ -45,29 +45,38 @@ def silo( raise NotImplementedError -def extract_integration_sources( +def extract_band_sources( path_to_trial_table: Optional[Union[str, Path]] = None, - params_in: Optional[dict] = None, -) -> None: - """Perform integration (band-scan) source extraction for SLAP2. + params_in: Optional[Union[dict, SiloParams]] = None, +) -> "object": + """Perform band-scan source extraction for SLAP2. The band-scan variant of SILo: operates on superpixel / DMD-geometry - data and the integration lookup table produced by - :func:`giant_python.pipeline.register.integration_registration` - (``fnAdataInt``), rather than on a reconstructed pixel movie. Shares the - peak-detection, baseline, and deconvolution kernels with :func:`silo`. - Corresponds to ``extractSLAP2IntegrationSources.py`` in - ophys-slap2-analysis. + data and the band lookup table produced by + :func:`giant_python.pipeline.register.band_registration` + (``fnAdataInt``), rather than on a reconstructed pixel movie. Corresponds + to ``extractSLAP2IntegrationSources.py`` in ophys-slap2-analysis. + + A thin adapter over :func:`giant_python.bandsilo.pipeline\ +.extract_band_sources` (imported lazily so ``import giant_python`` never + pulls in the heavy BandSILo dependencies). Parameters ---------- path_to_trial_table : str or Path, optional - Full path to the trial table file (with integration alignment data). - params_in : dict, optional - Parameter overrides. Keys and default values are defined in - :func:`giant_python.models.params.set_params`. + Full path to the ``trial_table.h5`` file (with band alignment + data). + params_in : dict or SiloParams, optional + Parameter overrides. + + Returns + ------- + ExperimentSummary + The extracted sources and summary (also written to disk). """ - raise NotImplementedError + from ..bandsilo.pipeline import extract_band_sources as _extract_impl + + return _extract_impl(path_to_trial_table, params_in) class SourceExtractor(Stage): @@ -84,6 +93,7 @@ class SourceExtractor(Stage): """ def __init__(self, params: Optional[SiloParams] = None) -> None: + """Store the extraction parameters (default :class:`SiloParams`).""" self.params = params or SiloParams() @staticmethod @@ -97,7 +107,7 @@ def for_scan_mode( ---------- scan_mode : str ``"standard"`` (-> :class:`StandardSourceExtractor`) or - ``"integration"`` (-> :class:`IntegrationSourceExtractor`). + ``"band"`` (-> :class:`BandSourceExtractor`). params : SiloParams, optional Source-extraction parameters. @@ -105,8 +115,17 @@ def for_scan_mode( ------- SourceExtractor The appropriate backend instance. + + Raises + ------ + ValueError + If *scan_mode* is not ``"standard"`` or ``"band"``. """ - raise NotImplementedError + if scan_mode == "band": + return BandSourceExtractor(params) + if scan_mode == "standard": + return StandardSourceExtractor(params) + raise ValueError(f"Unknown scan mode: {scan_mode!r}") def run(self, trial_table: "object") -> "object": """Extract sources and write experiment_summary.h5. @@ -132,12 +151,18 @@ def run(self, trial_table: "object") -> "object": raise NotImplementedError -class IntegrationSourceExtractor(SourceExtractor): - """Integration (band-scan) SILo backend. +class BandSourceExtractor(SourceExtractor): + """Band-scan SILo backend. - Wraps :func:`extract_integration_sources`. + Wraps :func:`extract_band_sources`. """ def run(self, trial_table: "object") -> "object": - """Extract from integration data. See :meth:`SourceExtractor.run`.""" - raise NotImplementedError + """Extract from band data. See :meth:`SourceExtractor.run`. + + Resolves the on-disk ``trial_table.h5`` path from the trial table's + ``savedr`` and delegates to :func:`extract_band_sources` with + this extractor's parameters. + """ + path = Path(trial_table.savedr) / "trial_table.h5" + return extract_band_sources(path, self.params) diff --git a/src/giant_python/pipeline/organize.py b/src/giant_python/pipeline/organize.py index 1f59ef0..97d6492 100644 --- a/src/giant_python/pipeline/organize.py +++ b/src/giant_python/pipeline/organize.py @@ -117,6 +117,7 @@ class TrialTableBuilder(Stage): """ def __init__(self, microscope: str = "slap2") -> None: + """Store the microscope kind (``"slap2"`` or ``"bergamo"``).""" self.microscope = microscope def run( diff --git a/src/giant_python/pipeline/pipeline.py b/src/giant_python/pipeline/pipeline.py index 230bf1b..fab5625 100644 --- a/src/giant_python/pipeline/pipeline.py +++ b/src/giant_python/pipeline/pipeline.py @@ -54,6 +54,7 @@ def __init__( align_params: Optional[AlignParams] = None, silo_params: Optional[SiloParams] = None, ) -> None: + """Store session config; ``tt``/``summary`` start empty.""" self.microscope = microscope self.save_dir = Path(save_dir) self.align_params = align_params or AlignParams() diff --git a/src/giant_python/pipeline/register.py b/src/giant_python/pipeline/register.py index 70d1a72..4c865c3 100644 --- a/src/giant_python/pipeline/register.py +++ b/src/giant_python/pipeline/register.py @@ -2,8 +2,8 @@ Exposes the porting-faithful registration functions plus a :class:`MotionCorrector` ABC with one backend per MATLAB registration variant -(MultiRoi / Strip / Band). Band-mode registration is exposed as -:func:`integration_registration` / :class:`IntegrationRegistration` (the +(MultiRoi / Strip / Band). Band-scan registration is exposed as +:func:`band_registration` / :class:`BandRegistration` (the Python API name) and corresponds to ``BandRegistration.m``, writing ``bandRegLookupTable.h5``. """ @@ -66,11 +66,11 @@ def strip_registration( raise NotImplementedError -def integration_registration( +def band_registration( full_path_to_trial_table: Optional[Union[str, Path]] = None, params_in: Optional[dict] = None, ) -> None: - """Perform band / integration-mode motion correction for SLAP2 recordings. + """Perform band-scan motion correction for SLAP2 recordings. Builds the cached ``motion_correction/bandRegLookupTable.h5`` lookup table (on first run) and writes per-trial ``*_ALIGNMENTDATA.h5`` files including @@ -102,6 +102,7 @@ class MotionCorrector(Stage): """ def __init__(self, params: Optional[AlignParams] = None) -> None: + """Store the alignment parameters (default :class:`AlignParams`).""" self.params = params or AlignParams() @staticmethod @@ -158,14 +159,14 @@ def run(self, trial_table: "object") -> "object": raise NotImplementedError -class IntegrationRegistration(MotionCorrector): - """SLAP2 BandRegistration backend (wraps ``integration_registration``). +class BandRegistration(MotionCorrector): + """SLAP2 BandRegistration backend (wraps ``band_registration``). Corresponds to ``BandRegistration.m``; writes ``bandRegLookupTable.h5``. """ def run(self, trial_table: "object") -> "object": - """Register SLAP2 band/integration trials. + """Register SLAP2 band trials. See ``MotionCorrector.run``. """ diff --git a/tests/test_bandsilo_geometry.py b/tests/test_bandsilo_geometry.py index cf5dc47..1217b9b 100644 --- a/tests/test_bandsilo_geometry.py +++ b/tests/test_bandsilo_geometry.py @@ -38,7 +38,7 @@ class TestSubsampleMatrixInds(unittest.TestCase): """build_subsample_matrix_inds picks median open pixels.""" def test_median_reference_pixel(self): - """Odd-length superpixels use the median; even-length use the middle.""" + """Odd-length pixels use the median; even-length use the middle.""" all_ids = np.array([[1], [2]], dtype=np.int32) mask = np.array( [[10, 1], [12, 1], [14, 1], [20, 2], [22, 2]], dtype=np.int32 diff --git a/tests/test_bandsilo_peaks.py b/tests/test_bandsilo_peaks.py index c80f650..7721ff8 100644 --- a/tests/test_bandsilo_peaks.py +++ b/tests/test_bandsilo_peaks.py @@ -337,8 +337,7 @@ def test_exclusion_mask_3d(self): self._act_im(), peak_th=4.0, exclusion_mask=excl, buffer_size=3 ) z1_in_box = any( - int(z) == 1 and 20 <= y < 30 and 10 <= x < 20 - for z, y, x in seeds + int(z) == 1 and 20 <= y < 30 and 10 <= x < 20 for z, y, x in seeds ) self.assertFalse(z1_in_box) diff --git a/tests/test_bandsilo_pipeline.py b/tests/test_bandsilo_pipeline.py new file mode 100644 index 0000000..7ea152f --- /dev/null +++ b/tests/test_bandsilo_pipeline.py @@ -0,0 +1,238 @@ +"""Tests for giant_python.bandsilo.pipeline output assembly + writing. + +Covers the pure/testable Phase-8 surface: per-path output assembly, the +byte-compatible ``experiment_summary.h5`` writer, and the returned +``ExperimentSummary`` builder. The full orchestration driver +(``extract_band_sources`` / ``_process_dmd``) reads SLAP2 files, runs +the torch NMF, and does disk IO end-to-end, so it is not unit-tested here (its +Phase 1-7 kernels are each covered/cross-checked in their own modules). +""" + +import os +import tempfile +import unittest + +import h5py +import numpy as np +import torch + +from giant_python.bandsilo import pipeline as pl + + +def _trial(n_frames, n_sources, n_channels, n_soma): + """One synthetic get_high_res_traces 8-tuple.""" + rng = np.random.default_rng(n_frames) + return ( + rng.random((n_frames, n_sources)).astype(np.float32), # phi (dF_ls) + (rng.random((n_frames, n_sources)) + 5).astype(np.float32), # F0 + np.arange(n_frames, dtype=float), # frames + np.arange(3), # sel_pix_idxs + rng.random((n_frames, n_channels)).astype(np.float32), # global_f + (rng.random(n_frames), rng.random(n_frames), rng.random(n_frames)), + ( + np.zeros(n_frames, np.int16), + np.zeros(n_frames, np.int16), + np.zeros(n_frames, np.int16), + ), + rng.random((n_frames, n_soma, n_channels)).astype(np.float32), + ) + + +def _path_result( + n_sources=2, + n_channels=1, + nz=1, + npc=4, + npr=5, + draw_user_rois=False, + soma_masks=None, + soma_labels=None, + n_soma=0, +): + """Build a synthetic PathResult for one DMD path.""" + n_pixels = nz * npc * npr + results = [ + _trial(6, n_sources, n_channels, n_soma), + _trial(7, n_sources, n_channels, n_soma), + ] + return pl.PathResult( + a=torch.rand((n_pixels, n_sources)), + source_params=torch.rand((n_sources, 6)), + source_snr=np.arange(1, n_sources + 1, dtype=np.float32), + act_im=np.random.rand(nz, npc, npr).astype(np.float32), + mean_im=np.random.rand(n_channels, nz, npc, npr).astype(np.float32), + act_im_peaks=np.zeros((n_sources, 3)), + trace_results=results, + z_depths=np.zeros((nz, 1), dtype=np.int32), + num_fast_zs=nz, + dmd_pixels_per_column=npc, + dmd_pixels_per_row=npr, + num_channels=n_channels, + denoise_window=3, + baseline_window=5, + draw_user_rois=draw_user_rois, + soma_masks=soma_masks, + soma_labels=soma_labels, + yx_shape=(npc, npr), + ) + + +class TestWithNanChannel(unittest.TestCase): + """_with_nan_channel reshapes and NaN-fills non-activity channels.""" + + def test_shape_and_channels(self): + """(frames, sources) -> (sources, channels, frames); ch>0 is NaN.""" + arr = np.arange(6, dtype=np.float32).reshape(3, 2) # 3 frames, 2 src + out = pl._with_nan_channel(arr, 2) + self.assertEqual(out.shape, (2, 2, 3)) + np.testing.assert_array_equal(out[:, 0, :], arr.T) + self.assertTrue(np.all(np.isnan(out[:, 1, :]))) + + +class TestAssemblePathOutputs(unittest.TestCase): + """assemble_path_outputs shapes and content.""" + + def test_shapes(self): + """Every dataset has the schema shape.""" + pr = _path_result(n_sources=2, n_channels=1, nz=1, npc=4, npr=5) + asm = pl.assemble_path_outputs(pr) + self.assertEqual(asm["profiles"].shape, (2, 1, 4, 5)) + self.assertEqual(asm["coords"].shape, (2, 3)) + self.assertEqual(asm["dF_ls"].shape, (2, 1, 13)) # 6+7 frames + self.assertEqual(asm["F0"].shape, (2, 1, 13)) + self.assertEqual(asm["SNR"].shape, (2, 1)) + self.assertEqual(asm["global_f"].shape, (1, 13)) + self.assertEqual(list(asm["trial_num_frames"].ravel()), [6, 7]) + self.assertEqual(asm["discard_frames"].shape, (13, 1)) + self.assertIsNone(asm["user_rois"]) + + def test_two_channels_global_shape(self): + """Two channels widen the global-F channel axis.""" + pr = _path_result(n_channels=2) + asm = pl.assemble_path_outputs(pr) + self.assertEqual(asm["global_f"].shape, (2, 13)) + self.assertEqual(asm["dF_ls"].shape[1], 2) + + def test_user_rois_assembled(self): + """draw_user_rois with masks assembles the user-ROI datasets.""" + masks = [np.ones((1, 4, 5), dtype=bool), np.zeros((1, 4, 5), bool)] + pr = _path_result( + draw_user_rois=True, + soma_masks=masks, + soma_labels=["SOMA", "SOMA2"], + n_soma=2, + ) + asm = pl.assemble_path_outputs(pr) + self.assertIsNotNone(asm["user_rois"]) + self.assertEqual(asm["user_rois"]["mask"].shape, (2, 1, 4, 5)) + # F: (rois, channels, total_frames) + self.assertEqual(asm["user_rois"]["F"].shape, (2, 1, 13)) + + def test_user_rois_empty_masklist(self): + """draw_user_rois with an empty mask list yields a 0-ROI mask array.""" + pr = _path_result( + draw_user_rois=True, soma_masks=[], soma_labels=[], n_soma=0 + ) + asm = pl.assemble_path_outputs(pr) + self.assertEqual(asm["user_rois"]["mask"].shape, (0, 1, 4, 5)) + + +class TestWriteExperimentSummary(unittest.TestCase): + """write_experiment_summary produces the byte-compatible schema.""" + + def _write_and_read(self, assembled_paths, params=None): + """Write then reopen, returning the h5 file handle's data as dict.""" + params = params or {"operator": "test", "numChannels": 1} + path = tempfile.mktemp(suffix=".h5") + pl.write_experiment_summary(path, params, assembled_paths) + return path + + def test_schema_and_values(self): + """Root flag/params + one Path group with the expected datasets.""" + asm = pl.assemble_path_outputs(_path_result()) + path = self._write_and_read([asm]) + try: + with h5py.File(path, "r") as f: + self.assertEqual(int(f["row_major"][()]), 1) + self.assertIn("params", f) + grp = f["Path1"] + self.assertEqual( + grp["sources/spatial/profiles"].shape, (2, 1, 4, 5) + ) + self.assertEqual( + grp["sources/temporal/dF_ls"].shape, (2, 1, 13) + ) + self.assertEqual(grp["global/F"].shape, (1, 13)) + for key in pl._FRAME_INFO_KEYS: + self.assertIn(key, grp["frame_info"]) + self.assertIn("act_im", grp["visualizations"]) + self.assertNotIn("user_rois", grp) + finally: + os.remove(path) + + def test_user_rois_written(self): + """The user_rois group is written when soma ROIs are present.""" + masks = [np.ones((1, 4, 5), dtype=bool)] + pr = _path_result( + draw_user_rois=True, + soma_masks=masks, + soma_labels=["SOMA"], + n_soma=1, + ) + path = self._write_and_read([pl.assemble_path_outputs(pr)]) + try: + with h5py.File(path, "r") as f: + ur = f["Path1/user_rois"] + self.assertEqual(ur["mask"].shape, (1, 1, 4, 5)) + self.assertEqual(ur["F"].shape, (1, 1, 13)) + self.assertEqual(ur["labels"].shape, (1, 1)) + finally: + os.remove(path) + + def test_multi_path_and_overwrite(self): + """Two paths write Path1/Path2; re-writing overwrites in place.""" + asm = [pl.assemble_path_outputs(_path_result()) for _ in range(2)] + path = self._write_and_read(asm) + try: + # second write to the same file overwrites the Path groups + pl.write_experiment_summary(path, {"operator": "test"}, asm) + with h5py.File(path, "r") as f: + self.assertIn("Path1", f) + self.assertIn("Path2", f) + finally: + os.remove(path) + + +class TestBuildExperimentSummary(unittest.TestCase): + """build_experiment_summary populates the returned dataclass.""" + + def test_sources_and_visualizations(self): + """Per-path sources are populated with profile/coords/traces/SNR.""" + asm = pl.assemble_path_outputs(_path_result(n_sources=2)) + summary = pl.build_experiment_summary({"operator": "t"}, [asm]) + self.assertEqual(len(summary.sources), 1) + self.assertEqual(len(summary.sources[0]), 2) + src = summary.sources[0][0] + self.assertEqual(src.profile.shape, (1, 4, 5)) + self.assertEqual(src.snr, 1.0) + self.assertIsNotNone(summary.visualizations.act_im) + self.assertEqual(summary.user_rois, [[]]) + self.assertIn("trial_num_frames", summary.frame_info) + + def test_user_rois_populated(self): + """User ROIs appear in the summary when present.""" + masks = [np.ones((1, 4, 5), dtype=bool)] + pr = _path_result( + draw_user_rois=True, + soma_masks=masks, + soma_labels=["SOMA"], + n_soma=1, + ) + asm = pl.assemble_path_outputs(pr) + summary = pl.build_experiment_summary({"operator": "t"}, [asm]) + self.assertEqual(len(summary.user_rois[0]), 1) + self.assertEqual(summary.user_rois[0][0].label, "SOMA") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_bandsilo_trial_data.py b/tests/test_bandsilo_trial_data.py index 4c260aa..b034b65 100644 --- a/tests/test_bandsilo_trial_data.py +++ b/tests/test_bandsilo_trial_data.py @@ -25,6 +25,8 @@ def __init__(self, lines_per_cycle, num_cycles, line_specs, channels=2): self._channels = channels class _Meta: + """Minimal metadata stub exposing ``linePeriod_s``.""" + linePeriod_s = 1.0 self.metaData = _Meta() @@ -174,12 +176,12 @@ def test_line_with_no_matching_superpixels_skipped(self): self.assertGreater(acc["data"][1].sum(), 0) -class TestReadIntegrationTrialData(unittest.TestCase): - """read_integration_trial_data orchestrates open+accumulate+alignment.""" +class TestReadBandTrialData(unittest.TestCase): + """read_band_trial_data orchestrates open+accumulate+alignment.""" def test_skipped_trial(self): """keep_trial=False returns None without opening anything.""" - out = td.read_integration_trial_data( + out = td.read_band_trial_data( 0, False, 0, 100.0, np.zeros((1, 1)), "/data", {}, 1 ) self.assertIsNone(out) @@ -206,7 +208,7 @@ def test_full_read_with_monkeypatched_open(self): "motionDSz": np.zeros(2), } try: - out = td.read_integration_trial_data( + out = td.read_band_trial_data( 0, True, 0, diff --git a/tests/test_io.py b/tests/test_io.py index cbc56e7..8c537e6 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -7,7 +7,7 @@ from giant_python.io import ( get_online_motion, load_struct_h5, - read_integration_trial_data, + read_band_trial_data, ref_pixs_to_drc, save_struct_h5, scanimagetiff_data_wrapper, @@ -44,7 +44,7 @@ def test_scanimagetiff_data_wrapper(self): class TestSlap2(unittest.TestCase): - """Tests for SLAP2 readers (online motion + integration).""" + """Tests for SLAP2 readers (online motion + band).""" def test_get_online_motion(self): """get_online_motion raises NotImplementedError.""" @@ -56,10 +56,10 @@ def test_ref_pixs_to_drc(self): with self.assertRaises(NotImplementedError): ref_pixs_to_drc(np.arange(10), 4, 4) - def test_read_integration_trial_data(self): - """read_integration_trial_data raises NotImplementedError.""" + def test_read_band_trial_data(self): + """read_band_trial_data raises NotImplementedError.""" with self.assertRaises(NotImplementedError): - read_integration_trial_data(None, np.arange(5), np.arange(3)) + read_band_trial_data(None, np.arange(5), np.arange(3)) if __name__ == "__main__": diff --git a/tests/test_math.py b/tests/test_math.py index d11829c..67762ac 100644 --- a/tests/test_math.py +++ b/tests/test_math.py @@ -5,18 +5,16 @@ import numpy as np from giant_python.math import ( + compute_f0, + detect_peaks_2d, dft_register_clipped, + gaussian_peaks_integrated, + get_act_im_peaks, interp_frame, xcorr2_nans, xcorr2_nans3d, xcorr2_nans_weighted, ) -from giant_python.math import ( - compute_f0, - detect_peaks_2d, - gaussian_peaks_integrated, - get_act_im_peaks, -) from giant_python.math.deconv import deconvolve_trace from giant_python.math.nmf import nmf_decompose from giant_python.math.variance import activity_image diff --git a/tests/test_package_metadata.py b/tests/test_package_metadata.py index b4fc5c7..4b89eb4 100644 --- a/tests/test_package_metadata.py +++ b/tests/test_package_metadata.py @@ -4,9 +4,11 @@ import unittest from pathlib import Path - PACKAGE_INIT = ( - Path(__file__).resolve().parents[1] / "src" / "giant_python" / "__init__.py" + Path(__file__).resolve().parents[1] + / "src" + / "giant_python" + / "__init__.py" ) diff --git a/tests/test_parallel.py b/tests/test_parallel.py index 9932695..0992917 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -6,13 +6,29 @@ from giant_python.parallel import map_trials +def _double(x): + """Module-level (picklable) worker for the parallel path.""" + return x * 2 + + class TestMapTrials(unittest.TestCase): """Tests for the per-trial parallelism helper.""" - def test_not_implemented(self): - """map_trials raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - map_trials(lambda x: x, [1, 2, 3]) + def test_serial(self): + """n_workers=1 runs serially, preserving order.""" + self.assertEqual( + map_trials(_double, [1, 2, 3], n_workers=1), [2, 4, 6] + ) + + def test_single_trial_runs_serial(self): + """A single trial short-circuits to the serial path.""" + self.assertEqual(map_trials(_double, [5], n_workers=4), [10]) + + def test_parallel(self): + """Multiple workers fan out and return results in input order.""" + self.assertEqual( + map_trials(_double, [1, 2, 3, 4], n_workers=2), [2, 4, 6, 8] + ) class TestCli(unittest.TestCase): diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index b3b7289..7d6da72 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -1,11 +1,12 @@ """Tests for the pipeline subpackage (stage functions, classes, facade).""" import unittest +from unittest import mock from giant_python.models import AlignParams, SiloParams from giant_python.pipeline import ( - IntegrationRegistration, - IntegrationSourceExtractor, + BandRegistration, + BandSourceExtractor, MotionCorrector, MultiRoiRegistration, Pipeline, @@ -13,10 +14,9 @@ StandardSourceExtractor, StripRegistration, TrialTableBuilder, + band_registration, build_trial_table, build_trial_table_slap2, - extract_integration_sources, - integration_registration, multi_roi_registration, silo, strip_registration, @@ -52,21 +52,16 @@ def test_strip_registration(self): with self.assertRaises(NotImplementedError): strip_registration() - def test_integration_registration(self): - """integration_registration raises NotImplementedError.""" + def test_band_registration(self): + """band_registration raises NotImplementedError.""" with self.assertRaises(NotImplementedError): - integration_registration() + band_registration() def test_silo(self): """silo raises NotImplementedError.""" with self.assertRaises(NotImplementedError): silo() - def test_extract_integration_sources(self): - """extract_integration_sources raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - extract_integration_sources() - class TestStageClasses(unittest.TestCase): """Stage classes construct; their run methods raise NotImplementedError.""" @@ -92,7 +87,7 @@ def test_registration_backends_run(self): for backend in ( MultiRoiRegistration, StripRegistration, - IntegrationRegistration, + BandRegistration, ): with self.assertRaises(NotImplementedError): backend().run(None) @@ -104,18 +99,51 @@ def test_source_extractor(self): extractor.run(None) def test_source_extractor_for_scan_mode(self): - """SourceExtractor.for_scan_mode raises NotImplementedError.""" + """for_scan_mode dispatches by scan mode and rejects unknown modes.""" + self.assertIsInstance( + SourceExtractor.for_scan_mode("band", SiloParams()), + BandSourceExtractor, + ) + self.assertIsInstance( + SourceExtractor.for_scan_mode("standard", SiloParams()), + StandardSourceExtractor, + ) + with self.assertRaises(ValueError): + SourceExtractor.for_scan_mode("bogus", SiloParams()) + + def test_standard_backend_run_not_implemented(self): + """The standard backend's run is still a stub.""" with self.assertRaises(NotImplementedError): - SourceExtractor.for_scan_mode("integration", SiloParams()) + StandardSourceExtractor().run(None) - def test_extractor_backends_run(self): - """Each extraction backend's run raises NotImplementedError.""" - for backend in ( - StandardSourceExtractor, - IntegrationSourceExtractor, - ): - with self.assertRaises(NotImplementedError): - backend().run(None) + def test_band_backend_run_delegates(self): + """The band backend resolves the trial-table path and delegates. + + The heavy driver is patched so only the wiring is exercised. + """ + import giant_python.bandsilo.pipeline as bpl + from giant_python.models import TrialTable + + captured = {} + + def _fake(path, params_in=None): + """Capture the delegated call and return a sentinel.""" + captured["path"] = path + captured["params"] = params_in + return "SUMMARY" + + orig = bpl.extract_band_sources + bpl.extract_band_sources = _fake + try: + params = SiloParams(scan_mode="band") + extractor = BandSourceExtractor(params) + out = extractor.run(TrialTable(savedr="/results")) + finally: + bpl.extract_band_sources = orig + + self.assertEqual(out, "SUMMARY") + self.assertEqual(captured["path"].name, "trial_table.h5") + self.assertIs(captured["params"], params) class TestPipelineFacade(unittest.TestCase): @@ -133,19 +161,43 @@ def test_holds_session_state(self): self.assertEqual(self.pipe.silo_params.microscope, "slap2") def test_organize_delegates(self): - """organize delegates to the (unimplemented) builder.""" - with self.assertRaises(NotImplementedError): - self.pipe.organize("/data") + """organize delegates to the builder and stores its trial table.""" + with mock.patch( + "giant_python.pipeline.pipeline.TrialTableBuilder" + ) as builder: + builder.return_value.run.return_value = "TT" + result = self.pipe.organize("/data") + self.assertIs(result, self.pipe) + self.assertEqual(self.pipe.tt, "TT") def test_register_delegates(self): - """register delegates to the (unimplemented) corrector.""" - with self.assertRaises(NotImplementedError): - self.pipe.register() + """register delegates to the motion corrector and updates the table.""" + with mock.patch( + "giant_python.pipeline.pipeline.MotionCorrector" + ) as corrector: + corrector.for_microscope.return_value.run.return_value = "TT2" + self.pipe.tt = "TT" + result = self.pipe.register() + self.assertIs(result, self.pipe) + self.assertEqual(self.pipe.tt, "TT2") + + def test_annotate_delegates(self): + """annotate lazily invokes the ROI-annotation GUI step.""" + with mock.patch("giant_python.gui.annotate_rois") as annotate_rois: + result = self.pipe.annotate() + annotate_rois.assert_called_once() + self.assertIs(result, self.pipe) def test_extract_delegates(self): - """extract delegates to the (unimplemented) extractor.""" - with self.assertRaises(NotImplementedError): - self.pipe.extract() + """extract dispatches by scan mode and stores the summary.""" + self.pipe.silo_params.scan_mode = "band" + with mock.patch( + "giant_python.pipeline.pipeline.SourceExtractor" + ) as extractor: + extractor.for_scan_mode.return_value.run.return_value = "SUMMARY" + result = self.pipe.extract() + self.assertIs(result, self.pipe) + self.assertEqual(self.pipe.summary, "SUMMARY") def test_run_all(self): """run_all raises NotImplementedError.""" From 4f1dcfa2879b9c908b07d3faedc99ec7948a91da Mon Sep 17 00:00:00 2001 From: Michael Xie Date: Thu, 6 Aug 2026 16:50:04 -0700 Subject: [PATCH 12/55] separate out annotations and fix trialtable --- src/giant_python/bandsilo/annotate.py | 331 +++++++++++++++++++++++++ src/giant_python/bandsilo/hdf5.py | 160 +++++++++--- src/giant_python/bandsilo/pipeline.py | 119 ++++++++- src/giant_python/cli.py | 146 ++++++++++- src/giant_python/io/hdf5.py | 97 +++++++- src/giant_python/models/params.py | 8 + src/giant_python/models/trial_table.py | 51 +++- src/giant_python/pipeline/pipeline.py | 26 +- tests/test_bandsilo_annotate.py | 151 +++++++++++ tests/test_bandsilo_hdf5.py | 79 +++++- tests/test_bandsilo_pipeline.py | 33 +++ tests/test_cli.py | 99 ++++++++ tests/test_io.py | 56 ++++- tests/test_models.py | 62 ++++- tests/test_parallel.py | 12 +- tests/test_pipeline.py | 49 +++- 16 files changed, 1398 insertions(+), 81 deletions(-) create mode 100644 src/giant_python/bandsilo/annotate.py create mode 100644 tests/test_bandsilo_annotate.py create mode 100644 tests/test_cli.py diff --git a/src/giant_python/bandsilo/annotate.py b/src/giant_python/bandsilo/annotate.py new file mode 100644 index 0000000..ddb3feb --- /dev/null +++ b/src/giant_python/bandsilo/annotate.py @@ -0,0 +1,331 @@ +"""Standalone band-scan ROI annotation step (decoupled from extraction). + +The GIAnT-MATLAB workflow runs ROI annotation as a *separate* CodeOcean +capsule (``run_capsule`` + ``annotateROIs.m`` / +``loadSessionImagesForAnnotation``) that writes an ``annotations.h5`` the +extraction step later consumes. This +module is the BandSILo analog: :func:`annotate_band_rois` loads just enough of +a ``trial_table.h5`` to display the per-DMD reference images (no +background/rho/NMF compute), launches the ROI selector, and writes +``/annotations/annotations.h5``. + +The same load-or-draw seam (:func:`resolve_user_rois`) is reused by the +headless extraction driver, so annotation geometry is computed identically in +both places. Extraction never draws unless it resolves to an interactive +session (:func:`resolve_interactivity`); otherwise it fails fast with guidance. + +The interactive GUI and disk/``slap2_utils`` IO cannot run headless and are +excluded from coverage; the pure helpers (first-valid-trial, motion medians, +interactivity resolution, and the load/fail-fast branches of +:func:`resolve_user_rois`) are unit-tested. +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path +from typing import Optional, Union + +import numpy as np + +from ..models.params import SiloParams +from . import geometry as geo +from .gui import ( + compute_user_roi_geometry, + load_annotations_h5, + run_user_roi_selection, + save_annotations_h5, +) +from .hdf5 import compute_keep_trials, load_alignment_data_h5, load_trial_table + + +def first_valid_trial(keep_trials: np.ndarray, dmd_ix: int) -> int: + """Return the first kept trial index for a DMD. + + Parameters + ---------- + keep_trials : ndarray of bool, shape (n_dmds, n_trials) + Per-DMD/-trial keep mask. + dmd_ix : int + 0-based DMD index. + + Returns + ------- + int + 0-based index of the first ``True`` entry in ``keep_trials[dmd_ix]``. + + Raises + ------ + ValueError + If no trial is kept for this DMD. + """ + row = np.asarray(keep_trials)[dmd_ix] + idxs = np.flatnonzero(row) + if idxs.size == 0: + raise ValueError(f"No valid trials for DMD {dmd_ix + 1}") + return int(idxs[0]) + + +def motion_median_from_adata(a_data: dict) -> tuple: + """Return the median ``(row, column, z)`` motion from alignment data. + + Mirrors the ``motion_medians`` computation in + ``extractSLAP2IntegrationSources.py``: the per-axis median of the rounded + downsampled motion estimates for one trial's alignment data. Both the + annotate step (drawing) and the extract step (loading) use this so the ROI + geometry is identical. + + Parameters + ---------- + a_data : dict + Alignment data from :func:`giant_python.bandsilo.hdf5\ +.load_alignment_data_h5` (``motionDSr``/``motionDSc``/``motionDSz``). + + Returns + ------- + tuple of int + ``(median_row, median_column, median_z)``. + """ + return ( + int(np.nanmedian(np.round(a_data["motionDSr"]))), + int(np.nanmedian(np.round(a_data["motionDSc"]))), + int(np.nanmedian(np.round(a_data["motionDSz"]))), + ) + + +def _stdin_is_tty() -> bool: # pragma: no cover - depends on runtime stdin + """Return True if stdin is attached to an interactive terminal.""" + return bool(getattr(sys.stdin, "isatty", lambda: False)()) + + +def resolve_interactivity(override: Optional[bool] = None) -> bool: + """Decide whether a drawing GUI may be launched in this process. + + Resolution order: an explicit ``override`` wins; otherwise the + ``GIANT_HEADLESS`` environment variable forces headless; otherwise fall + back to whether stdin is a TTY. A display probe (e.g. ``DISPLAY`` on Linux) + can be added here later without changing callers. + + Parameters + ---------- + override : bool or None + ``SiloParams.interactive`` (``True``/``False`` force the answer; + ``None`` auto-detects). + + Returns + ------- + bool + True if the process may open the ROI-drawing GUI. + """ + if override is not None: + return bool(override) + if os.environ.get("GIANT_HEADLESS"): + return False + return _stdin_is_tty() + + +def _empty_user_rois(n_dmds: int) -> dict: + """Return an all-empty per-DMD user-ROI selection for ``n_dmds`` paths.""" + keys = [f"DMD{d + 1}" for d in range(n_dmds)] + return { + "user_roi_masks": {k: [] for k in keys}, + "user_roi_superpixels": {k: [] for k in keys}, + "user_roi_labels": {k: [] for k in keys}, + "roi_records": {k: [] for k in keys}, + "annotated": False, + } + + +def resolve_user_rois( + annotation_dir: Union[str, Path], + n_dmds: int, + user_roi_geo: dict, + *, + interactive: bool, + ref_files: Optional[dict] = None, +) -> dict: + """Load ``annotations.h5`` if present, else draw (interactive) or fail. + + This is the shared seam between the standalone annotate step and the + headless extract step. When ``annotations.h5`` holds at least one ROI it is + loaded and returned. Otherwise, if ``interactive`` the drawing GUI runs and + the result is saved; if not, a :class:`RuntimeError` is raised telling the + user to run ``giant annotate`` first (so batch/cluster ``extract`` jobs + never block on a GUI). + + Parameters + ---------- + annotation_dir : str or Path + Directory holding (or to hold) ``annotations.h5``. + n_dmds : int + Number of DMD paths. + user_roi_geo : dict + Per-DMD geometry from :func:`compute_user_roi_geometry`. + interactive : bool + Whether a GUI may be launched (from :func:`resolve_interactivity`). + ref_files : dict, optional + ``{"DMD{N}": reference_file}`` provenance written into the file. + + Returns + ------- + dict + ``{"user_roi_masks", "user_roi_superpixels", "user_roi_labels", + "roi_records", "annotated"}``. + + Raises + ------ + RuntimeError + If drawing is required but the process is not interactive. + """ + path = os.path.join(str(annotation_dir), "annotations.h5") + skip, masks, superpixels, labels, records = load_annotations_h5( + path, n_dmds, user_roi_geo + ) + if skip: + return { + "user_roi_masks": masks, + "user_roi_superpixels": superpixels, + "user_roi_labels": labels, + "roi_records": records, + "annotated": True, + } + if not interactive: + raise RuntimeError( + "draw_user_rois is set but no usable ROIs were found at " + f"{path!r}, and this process is not interactive. Run " + "`giant annotate ` first, or run in an " + "interactive session (or pass interactive=True) to draw now." + ) + return _draw_and_save_user_rois( + annotation_dir, n_dmds, user_roi_geo, ref_files + ) + + +def _draw_and_save_user_rois( + annotation_dir: Union[str, Path], + n_dmds: int, + user_roi_geo: dict, + ref_files: Optional[dict], +) -> dict: # pragma: no cover - interactive GUI + disk IO + """Run the ROI selector, persist ``annotations.h5``, and return it.""" + os.makedirs(str(annotation_dir), exist_ok=True) + selection = run_user_roi_selection(user_roi_geo, n_dmds) + save_annotations_h5( + str(annotation_dir), + selection["roi_records"], + selection["user_roi_masks"], + n_dmds, + ref_files=ref_files, + ) + selection["annotated"] = True + return selection + + +def build_user_roi_geometry( + trial_table: dict, + lookup: dict, +) -> tuple: # pragma: no cover - reference-stack/aData disk IO + """Build per-DMD user-ROI geometry and reference files for a session. + + Loads each DMD's reference stack, superpixel index map, and + first-valid-trial motion medians, then derives the display geometry via + :func:`compute_user_roi_geometry`. Mirrors the ``soma_geo`` setup block of + ``main`` in ``extractSLAP2IntegrationSources.py``. + + Parameters + ---------- + trial_table : dict + Normalized trial table (provides ``datadr``, ``ref_stack``, + ``keep_trials``, ``fn_adata``, ``n_dmds``). + lookup : dict + Band-registration lookup table from + :func:`giant_python.bandsilo.geometry.load_lookup_table`. + + Returns + ------- + tuple + ``(user_roi_geo, ref_files)`` dicts keyed ``DMD{N}``. + """ + user_roi_geo, ref_files = {}, {} + datadr = str(trial_table["datadr"]) + keep_trials = trial_table["keep_trials"] + for dmd_ix in range(int(trial_table["n_dmds"])): + key = f"DMD{dmd_ix + 1}" + ref_stack, _, ref_file = geo.load_reference_stack( + datadr, trial_table["ref_stack"], dmd_ix + ) + smi = geo.build_subsample_matrix_inds( + lookup["allSuperPixelIDs"][key], lookup["sparseMaskInds"][key] + ) + fvt = first_valid_trial(keep_trials, dmd_ix) + a_data = load_alignment_data_h5(trial_table["fn_adata"][dmd_ix, fvt]) + user_roi_geo[key] = compute_user_roi_geometry( + ref_stack, + lookup["fastZ2RefZ"][key], + smi, + motion_median_from_adata(a_data), + ) + ref_files[key] = ref_file + return user_roi_geo, ref_files + + +def _resolve_params( + params_in, +) -> SiloParams: # pragma: no cover - trivial glue + """Coerce ``params_in`` (None / dict / SiloParams) to a SiloParams.""" + if params_in is None: + return SiloParams(scan_mode="band") + if isinstance(params_in, SiloParams): + return params_in + return SiloParams(**params_in) + + +def annotate_band_rois( + path_to_trial_table: Union[str, Path], + params_in=None, +) -> str: # pragma: no cover - full GUI + reference-stack/aData IO + """Annotate user ROIs for a band-scan session (standalone step). + + The BandSILo analog of the GIAnT-MATLAB annotation capsule: loads the + ``trial_table.h5`` and per-DMD reference geometry (no extraction compute), + then loads an existing ``annotations.h5`` or launches the ROI selector and + writes ``/annotations/annotations.h5``. Idempotent -- a session + that already has annotations short-circuits (matching the MATLAB + ``if exist(fnAnnH5)`` guard). + + Parameters + ---------- + path_to_trial_table : str or Path + Path to the ``trial_table.h5`` produced by BandRegistration. + params_in : SiloParams or dict, optional + Parameter overrides (``interactive`` gates the GUI). + + Returns + ------- + str + Path to the written (or existing) ``annotations.h5``. + """ + params = _resolve_params(params_in) + trial_table = load_trial_table(path_to_trial_table) + trial_table["keep_trials"] = compute_keep_trials( + trial_table["fn_adata"], + trial_table["filename"], + trial_table["datadr"], + ) + annotation_dir = trial_table["annotation_save_dr"] + + lookup = geo.load_lookup_table( + Path(trial_table["moco_save_dr"]) / "bandRegLookupTable.h5", + trial_table["n_dmds"], + ) + user_roi_geo, ref_files = build_user_roi_geometry(trial_table, lookup) + + resolve_user_rois( + annotation_dir, + trial_table["n_dmds"], + user_roi_geo, + interactive=resolve_interactivity(params.interactive), + ref_files=ref_files, + ) + return os.path.join(str(annotation_dir), "annotations.h5") diff --git a/src/giant_python/bandsilo/hdf5.py b/src/giant_python/bandsilo/hdf5.py index bcbcb45..f95b9b4 100644 --- a/src/giant_python/bandsilo/hdf5.py +++ b/src/giant_python/bandsilo/hdf5.py @@ -23,6 +23,8 @@ import h5py import numpy as np +from ..models import TrialTable + def to_serializable(val): """Recursively convert numpy types/arrays to JSON-friendly Python values. @@ -265,11 +267,17 @@ def _resolve_fn_adata(fn_adata: np.ndarray, moco_save_dr: str) -> np.ndarray: def load_trial_table(path: Union[str, Path]) -> dict: - """Load and normalize a GIAnT ``trial_table.h5`` for BandSILo. - - Reads the trial table written by BandRegistration.m and assembles the - normalized structure consumed by the band backend (mirrors the - setup block of ``main`` in ``extractSLAP2IntegrationSources.py``). + """Resolve a registered SLAP2 band ``trial_table.h5`` to a runtime view. + + Faithful load is delegated to :meth:`giant_python.models.TrialTable\ +.from_h5` (which mirrors the README schema exactly); this function performs + only the *structural* resolution the band backend needs: it hoists the + per-trial arrays out of ``slap2_info`` / ``motion_correction``, resolves + ``fn_adata`` basenames to absolute paths under + ``savedr/motion_correction``, and derives the result subdirectories and + ``(n_dmds, n_trials)``. The disk-dependent pieces (kept-trial mask, + per-DMD ``alignHz``, channel count) are computed separately by + :func:`compute_keep_trials` / :func:`read_align_info`. Parameters ---------- @@ -279,21 +287,31 @@ def load_trial_table(path: Union[str, Path]) -> dict: Returns ------- dict - Keys: ``datadr``, ``result_dr``, ``moco_save_dr``, + Flat runtime view with keys ``datadr``, ``savedr``, ``moco_save_dr``, ``annotation_save_dr``, ``src_extr_save_dr``, ``align_params``, - ``n_dmds``, ``n_trials``, and ``trial_table`` (a dict with - ``filename``, ``first_line``, ``last_line``, ``fn_adata`` (absolute - paths), ``datadr``, ``savedr``, ``ref_stack``). + ``n_dmds``, ``n_trials``, ``filename``, ``first_line``, ``last_line``, + ``fn_adata`` (absolute paths), and ``ref_stack``. + + Raises + ------ + ValueError + If the file is not a registered SLAP2 band trial table (missing + ``slap2_info`` or ``motion_correction/fn_adata``). """ - tt = load_struct_from_h5(path) - - datadr = str(tt["datadr"]) - result_dr = str(tt["savedr"]) + tt = TrialTable.from_h5(path) + slap2 = tt.slap2_info + motion_correction = tt.motion_correction or {} + if slap2 is None or "fn_adata" not in motion_correction: + raise ValueError( + "load_trial_table expects a registered SLAP2 band trial table " + "(slap2_info + motion_correction/fn_adata); " + f"got {str(path)!r}." + ) + + datadr = str(tt.datadr) if tt.datadr is not None else "" + result_dr = str(tt.savedr) if tt.savedr is not None else "" moco_save_dr = os.path.join(result_dr, "motion_correction") - motion_correction = tt.get("motion_correction", {}) or {} - slap2_info = tt.get("slap2_info", {}) or {} - align_params = { key: to_serializable(val) for key, val in ( @@ -301,34 +319,112 @@ def load_trial_table(path: Union[str, Path]) -> dict: ).items() } - filenames = np.atleast_2d(np.asarray(tt["filename"], dtype=object)) - first_line = np.atleast_2d(np.asarray(slap2_info["first_line"])) - last_line = np.atleast_2d(np.asarray(slap2_info["last_line"])) + filenames = np.atleast_2d(np.asarray(tt.filename, dtype=object)) + first_line = np.atleast_2d(np.asarray(slap2.first_line)) + last_line = np.atleast_2d(np.asarray(slap2.last_line)) fn_adata = np.atleast_2d( np.asarray(motion_correction["fn_adata"], dtype=object) ) n_dmds, n_trials = filenames.shape - fn_adata_full = _resolve_fn_adata(fn_adata, moco_save_dr) - - trial_table = { - "filename": filenames, - "first_line": first_line, - "last_line": last_line, - "fn_adata": fn_adata_full, - "datadr": datadr, - "savedr": result_dr, - "ref_stack": slap2_info.get("ref_stack", {}) or {}, - } return { "datadr": datadr, - "result_dr": result_dr, + "savedr": result_dr, "moco_save_dr": moco_save_dr, "annotation_save_dr": os.path.join(result_dr, "annotations"), "src_extr_save_dr": os.path.join(result_dr, "source_extraction"), "align_params": align_params, "n_dmds": int(n_dmds), "n_trials": int(n_trials), - "trial_table": trial_table, + "filename": filenames, + "first_line": first_line, + "last_line": last_line, + "fn_adata": _resolve_fn_adata(fn_adata, moco_save_dr), + "ref_stack": slap2.ref_stack or {}, } + + +def compute_keep_trials( + fn_adata_abs: np.ndarray, + filename: np.ndarray, + datadr: Union[str, Path], +) -> np.ndarray: + """Return the ``(n_dmds, n_trials)`` mask of trials whose files exist. + + A trial is kept only if its alignment-data file (absolute ``fn_adata`` + path) and its source file (``filename`` resolved under ``datadr``) both + exist on disk. Mirrors the ``verifyFiles`` / ``keepTrials`` logic in + ``extractSLAP2IntegrationSources.py``. + + Parameters + ---------- + fn_adata_abs : ndarray of object, shape (n_dmds, n_trials) + Absolute alignment-data paths (``''`` where absent). + filename : ndarray of object, shape (n_dmds, n_trials) + Source-file basenames, resolved under ``datadr``. + datadr : str or Path + Raw-data directory. + + Returns + ------- + ndarray of bool, shape (n_dmds, n_trials) + The kept-trial mask. + """ + fn_adata_abs = np.atleast_2d(np.asarray(fn_adata_abs, dtype=object)) + filename = np.atleast_2d(np.asarray(filename, dtype=object)) + n_dmds, n_trials = fn_adata_abs.shape + keep = np.ones((n_dmds, n_trials), dtype=bool) + for d in range(n_dmds): + for t in range(n_trials): + adata = fn_adata_abs[d, t] + src = filename[d, t] + src_path = os.path.join(str(datadr), str(src)) if src else "" + if ( + not adata + or not os.path.exists(str(adata)) + or not src + or not os.path.exists(src_path) + ): + keep[d, t] = False + return keep + + +def read_align_info( + fn_adata_abs: np.ndarray, + keep_trials: np.ndarray, + n_dmds: int, +) -> tuple: + """Read per-DMD ``alignHz`` and the channel count from alignment data. + + For each DMD, reads the first kept trial's ``_ALIGNMENTDATA.h5`` to recover + the alignment rate and acquisition channel count (which live in the + alignment files, not in ``trial_table.h5``). + + Parameters + ---------- + fn_adata_abs : ndarray of object, shape (n_dmds, n_trials) + Absolute alignment-data paths. + keep_trials : ndarray of bool, shape (n_dmds, n_trials) + Kept-trial mask from :func:`compute_keep_trials`. + n_dmds : int + Number of DMD paths. + + Returns + ------- + tuple + ``(align_hz, num_channels)`` -- a ``{"DMD{N}": alignHz}`` dict and the + channel count (``None`` if every DMD had no kept trials). + """ + fn_adata_abs = np.atleast_2d(np.asarray(fn_adata_abs, dtype=object)) + keep_trials = np.atleast_2d(np.asarray(keep_trials)) + align_hz = {} + num_channels = None + for d in range(n_dmds): + valid = np.flatnonzero(keep_trials[d]) + if valid.size == 0: + continue + a_data = load_alignment_data_h5(fn_adata_abs[d, valid[0]]) + align_hz[f"DMD{d + 1}"] = a_data["alignHz"] + num_channels = a_data["numChannels"] + return align_hz, num_channels diff --git a/src/giant_python/bandsilo/pipeline.py b/src/giant_python/bandsilo/pipeline.py index ce4f270..9b936f0 100644 --- a/src/giant_python/bandsilo/pipeline.py +++ b/src/giant_python/bandsilo/pipeline.py @@ -38,8 +38,19 @@ from . import geometry as geo from . import summary_images as si from . import trial_data as td +from .annotate import ( + build_user_roi_geometry, + resolve_interactivity, + resolve_user_rois, +) from .baseline import assemble_dff -from .hdf5 import load_trial_table, to_serializable, write_dict_to_h5group +from .hdf5 import ( + compute_keep_trials, + load_trial_table, + read_align_info, + to_serializable, + write_dict_to_h5group, +) from .nmf import fit_sources from .peaks import get_act_im_peaks from .traces import get_high_res_traces @@ -167,6 +178,37 @@ def _assemble_user_rois(pr: PathResult) -> Optional[dict]: return {"mask": masks_by_roi, "labels": pr.soma_labels, "F": f_soma_all} +def _dmd_user_rois(user_rois: Optional[dict], key: str) -> tuple: + """Return ``(soma_masks, soma_labels, soma_sps)`` for one DMD path. + + Slices a resolved session selection (from + :func:`giant_python.bandsilo.annotate.resolve_user_rois`) down to one DMD, + or the neutral ``(None, None, [])`` when user ROIs are disabled. + + Parameters + ---------- + user_rois : dict or None + The resolved session selection, or ``None`` when ``draw_user_rois`` + is unset. + key : str + The DMD key (``"DMD{N}"``). + + Returns + ------- + tuple + ``(soma_masks, soma_labels, soma_sps)`` -- the per-ROI masks and labels + (or ``None``) and the per-ROI superpixel index lists (``[]`` when + absent). + """ + if user_rois is None: + return None, None, [] + return ( + user_rois["user_roi_masks"].get(key), + user_rois["user_roi_labels"].get(key), + user_rois["user_roi_superpixels"].get(key, []), + ) + + def assemble_path_outputs(pr: PathResult) -> dict: """Assemble one path's HDF5-ready datasets from its results. @@ -394,6 +436,51 @@ def _resolve_params( return SiloParams(**params_in) +def _resolve_session_user_rois( + result_dr: str, + trial_table: dict, + lookup: dict, + params: SiloParams, +) -> Optional[ + dict +]: # pragma: no cover - reference-stack/aData IO + optional GUI + """Resolve per-DMD user ROIs for the session, or ``None`` if disabled. + + When ``draw_user_rois`` is set, builds the per-DMD annotation geometry and + loads ``/annotations/annotations.h5`` (drawing it interactively, + or failing fast when headless, via + :func:`giant_python.bandsilo.annotate.resolve_user_rois`). Extraction + stays non-interactive: drawing only happens as the resolver's fallback. + + Parameters + ---------- + result_dr : str + Session results directory (holds the ``annotations`` subdir). + trial_table : dict + Normalized trial table. + lookup : dict + Band-registration lookup table. + params : SiloParams + Extraction parameters (``draw_user_rois``/``interactive``). + + Returns + ------- + dict or None + The resolved user-ROI selection, or ``None`` when ``draw_user_rois`` + is unset. + """ + if not params.draw_user_rois: + return None + user_roi_geo, ref_files = build_user_roi_geometry(trial_table, lookup) + return resolve_user_rois( + Path(result_dr) / "annotations", + trial_table["n_dmds"], + user_roi_geo, + interactive=resolve_interactivity(params.interactive), + ref_files=ref_files, + ) + + def extract_band_sources( path_to_trial_table: Union[str, Path], params_in=None, @@ -434,11 +521,29 @@ def extract_band_sources( ) psf = geo.load_psf(params.psf_dilation, trial_table["n_dmds"]) + trial_table["keep_trials"] = compute_keep_trials( + trial_table["fn_adata"], trial_table["filename"], trial_table["datadr"] + ) + align_hz, num_channels = read_align_info( + trial_table["fn_adata"], + trial_table["keep_trials"], + trial_table["n_dmds"], + ) + trial_table["align_hz"] = align_hz + if params.num_channels is None: + params.num_channels = num_channels + + user_rois = _resolve_session_user_rois( + result_dr, trial_table, lookup, params + ) + assembled_paths = [] for dmd_ix in range(trial_table["n_dmds"]): assembled_paths.append( assemble_path_outputs( - _process_dmd(dmd_ix, trial_table, params, lookup, psf) + _process_dmd( + dmd_ix, trial_table, params, lookup, psf, user_rois + ) ) ) @@ -471,15 +576,19 @@ def _process_dmd( params: SiloParams, lookup: dict, psf: dict, + user_rois: Optional[dict] = None, ) -> PathResult: # pragma: no cover - full per-DMD IO/slap2/torch compute """Run Phases 3-7 for one DMD path and bundle the results. Reads the low-res trial data, estimates background/noise/rho, builds the activity image and detects peaks, localizes sources with NMF, and extracts high-res traces per trial (fanned out via - :func:`giant_python.parallel.map_trials`). + :func:`giant_python.parallel.map_trials`). When ``user_rois`` is provided, + this DMD's ROI masks/labels are attached and its per-ROI superpixels feed + the per-trial ``F_soma``. """ key = f"DMD{dmd_ix + 1}" + soma_masks, soma_labels, soma_sps = _dmd_user_rois(user_rois, key) datadr = str(trial_table["datadr"]) n_trials = trial_table["filename"].shape[1] num_channels = params.num_channels @@ -639,7 +748,7 @@ def _traces(trial_ix): dmd_pixels_per_column, dmd_pixels_per_row, num_channels, - [], + soma_sps, ) trace_results = map_trials(_traces, range(n_trials), params.max_workers) @@ -665,6 +774,8 @@ def _traces(trial_ix): np.ceil(params.baseline_window_s * params.analyze_hz) ), draw_user_rois=params.draw_user_rois, + soma_masks=soma_masks, + soma_labels=soma_labels, yx_shape=yx_shape, ) diff --git a/src/giant_python/cli.py b/src/giant_python/cli.py index 69580c7..e739ace 100644 --- a/src/giant_python/cli.py +++ b/src/giant_python/cli.py @@ -1,11 +1,140 @@ """Command-line interface: ``giant organize|register|annotate|extract``. Thin wrapper over the pipeline stages for batch / cluster use. Wired as the -``giant`` console script in pyproject.toml. +``giant`` console script in pyproject.toml. The ``annotate`` subcommand is the +standalone ROI-annotation step (the BandSILo analog of the GIAnT-MATLAB +annotation capsule): it writes ``annotations.h5``, which a later ``extract`` +run consumes without opening a GUI. + +The argument parser and the ``SiloParams`` builder are pure and unit-tested; +the individual command handlers drive heavy IO / GUI code and are excluded +from coverage. """ +from __future__ import annotations + +import argparse from typing import Optional, Sequence +from .models.params import SiloParams + + +def _add_silo_options(parser: argparse.ArgumentParser) -> None: + """Attach the shared source-extraction options to a subparser.""" + parser.add_argument( + "--scan-mode", + choices=("standard", "band"), + default="band", + help="SLAP2 scan mode selecting the extraction backend.", + ) + parser.add_argument( + "--draw-user-rois", + action="store_true", + help="Prompt for / use manual user ROIs.", + ) + interactive = parser.add_mutually_exclusive_group() + interactive.add_argument( + "--interactive", + dest="interactive", + action="store_true", + default=None, + help="Force the ROI-drawing GUI when annotations are missing.", + ) + interactive.add_argument( + "--headless", + dest="interactive", + action="store_false", + default=None, + help="Never open a GUI; fail fast if annotations are missing.", + ) + parser.add_argument( + "--operator", + default=SiloParams.operator, + help="Operator name recorded in the output metadata.", + ) + + +def build_parser() -> argparse.ArgumentParser: + """Build the ``giant`` argument parser. + + Returns + ------- + argparse.ArgumentParser + Parser with ``organize``, ``register``, ``annotate``, and ``extract`` + subcommands. Each subcommand stores its 0-based ``command`` name in the + parsed namespace. + """ + parser = argparse.ArgumentParser(prog="giant") + subparsers = parser.add_subparsers(dest="command") + + organize = subparsers.add_parser( + "organize", help="Build the trial table from raw data." + ) + organize.add_argument("data_dir", help="Raw recording directory.") + organize.add_argument("save_dir", help="Results directory.") + + register = subparsers.add_parser("register", help="Run motion correction.") + register.add_argument("trial_table", help="Path to trial_table.h5.") + + annotate = subparsers.add_parser( + "annotate", help="Draw / save user ROIs (standalone step)." + ) + annotate.add_argument("trial_table", help="Path to trial_table.h5.") + _add_silo_options(annotate) + + extract = subparsers.add_parser("extract", help="Run source extraction.") + extract.add_argument("trial_table", help="Path to trial_table.h5.") + _add_silo_options(extract) + + return parser + + +def _params_from_args(args: argparse.Namespace) -> SiloParams: + """Build a :class:`SiloParams` from parsed annotate/extract args.""" + return SiloParams( + scan_mode=args.scan_mode, + draw_user_rois=args.draw_user_rois, + interactive=args.interactive, + operator=args.operator, + ) + + +def _cmd_organize(args: argparse.Namespace) -> int: # pragma: no cover - IO + """Handle ``giant organize`` (delegates to the trial-table builder).""" + from .pipeline import TrialTableBuilder + + TrialTableBuilder("slap2").run(args.data_dir, args.save_dir) + return 0 + + +def _cmd_register(args: argparse.Namespace) -> int: # pragma: no cover - IO + """Handle ``giant register`` (delegates to motion correction).""" + from .pipeline import MotionCorrector + + MotionCorrector.for_microscope("slap2").run(args.trial_table) + return 0 + + +def _cmd_annotate( + args: argparse.Namespace, +) -> int: # pragma: no cover - GUI/IO + """Handle ``giant annotate`` (standalone ROI annotation).""" + from .bandsilo.annotate import annotate_band_rois + + annotate_band_rois(args.trial_table, _params_from_args(args)) + return 0 + + +def _cmd_extract(args: argparse.Namespace) -> int: # pragma: no cover - IO + """Handle ``giant extract`` (source extraction).""" + from .pipeline import SourceExtractor + + params = _params_from_args(args) + SourceExtractor.for_scan_mode(params.scan_mode, params).run( + args.trial_table + ) + return 0 + def main(argv: Optional[Sequence[str]] = None) -> int: """Entry point for the ``giant`` console script. @@ -21,9 +150,20 @@ def main(argv: Optional[Sequence[str]] = None) -> int: Returns ------- int - Process exit code. + Process exit code (``2`` when no subcommand is given). """ - raise NotImplementedError + parser = build_parser() + args = parser.parse_args(argv) + handlers = { + "organize": _cmd_organize, + "register": _cmd_register, + "annotate": _cmd_annotate, + "extract": _cmd_extract, + } + if args.command is None: + parser.print_help() + return 2 + return handlers[args.command](args) if __name__ == "__main__": # pragma: no cover diff --git a/src/giant_python/io/hdf5.py b/src/giant_python/io/hdf5.py index 44a2ee8..6b21e43 100644 --- a/src/giant_python/io/hdf5.py +++ b/src/giant_python/io/hdf5.py @@ -17,6 +17,92 @@ from pathlib import Path from typing import Union +import h5py +import numpy as np + + +def _is_string_dataset(ds: h5py.Dataset) -> bool: + """Return whether an h5py dataset holds (fixed/variable-length) strings.""" + if h5py.check_string_dtype(ds.dtype) is not None: + return True + return ds.dtype.kind in ("S", "U", "O") + + +def _decode_one(x) -> str: + """Decode a single bytes/str scalar to ``str``.""" + if isinstance(x, (bytes, np.bytes_)): + return x.decode("utf-8", "replace") + return str(x) + + +def _decode_strings(val): + """Decode a scalar or ndarray of (variable-length) strings to ``str``. + + Parameters + ---------- + val : bytes, str, or ndarray + Raw value read from a string dataset. + + Returns + ------- + str or ndarray of object + Decoded scalar string, or an object array of decoded strings. + """ + if isinstance(val, (bytes, str, np.bytes_, np.str_)): + return _decode_one(val) + arr = np.asarray(val, dtype=object) + if arr.size == 0: + return arr + return np.vectorize(_decode_one, otypes=[object])(arr) + + +def _read_dataset(ds: h5py.Dataset, row_major: bool): + """Read one dataset in MATLAB ``size()`` orientation. + + Scalar strings/paths (e.g. ``datadr``) are returned as plain ``str``; + string grids (e.g. ``filename``, ``fn_adata``) stay as object arrays with + MATLAB shape. Column-major files have their 2-D+ axes transposed to recover + the README orientation. + + Parameters + ---------- + ds : h5py.Dataset + The dataset to read. + row_major : bool + Whether the file was written row-major (``True``) or column-major. + + Returns + ------- + object + The decoded value (``str``, scalar, or ndarray). + """ + val = ds[()] + if _is_string_dataset(ds): + val = _decode_strings(val) + if isinstance(val, np.ndarray): + if (not row_major) and val.ndim >= 2: + val = val.T + if val.size == 1: + return val.reshape(-1)[0] + return val + val = np.asarray(val) + if (not row_major) and val.ndim >= 2: + val = val.T + return val + + +def _read_group(grp: h5py.Group, row_major: bool) -> dict: + """Read an h5py group into a nested dict (skipping ``row_major``).""" + out = {} + for key, item in grp.items(): + if key == "row_major": + continue + if isinstance(item, h5py.Group): + out[key] = _read_group(item, row_major) + else: + out[key] = _read_dataset(item, row_major) + return out + def load_struct_h5(path: Union[str, Path]) -> dict: """Load a (possibly nested) MATLAB-style struct from an HDF5 file. @@ -25,6 +111,11 @@ def load_struct_h5(path: Union[str, Path]) -> dict: ``row_major`` / axis-permutation conventions required for MATLAB compatibility. Corresponds to loadStructFromH5.m in GIAnT-MATLAB. + Groups become nested dicts, string datasets become ``str`` (scalars) or + object arrays (grids), and numeric datasets are returned in MATLAB + ``size()`` orientation. The ``/row_major`` flag (absent => column-major) + selects whether 2-D+ axes are transposed. + Parameters ---------- path : str or Path @@ -35,7 +126,11 @@ def load_struct_h5(path: Union[str, Path]) -> dict: dict Nested dictionary mirroring the HDF5 group hierarchy. """ - raise NotImplementedError + with h5py.File(path, "r") as f: + row_major = False + if "row_major" in f: + row_major = bool(np.asarray(f["row_major"][()]).reshape(-1)[0]) + return _read_group(f, row_major) def save_struct_h5(struct: dict, path: Union[str, Path]) -> None: diff --git a/src/giant_python/models/params.py b/src/giant_python/models/params.py index 0b4f849..5bfd328 100644 --- a/src/giant_python/models/params.py +++ b/src/giant_python/models/params.py @@ -74,6 +74,13 @@ class SiloParams: draw_user_rois : bool Whether to prompt for / use user ROIs; also gates writing the ``user_rois`` group and per-ROI traces to ``experiment_summary.h5``. + interactive : bool or None + Tri-state override for whether a GUI may be launched when + ``draw_user_rois`` is set but no ``annotations.h5`` exists. ``True`` + forces the drawing GUI, ``False`` forces headless (fail fast with + guidance), and ``None`` (the default) auto-detects (respecting the + ``GIANT_HEADLESS`` env var and stdin being a TTY). See + :func:`giant_python.bandsilo.annotate.resolve_interactivity`. analyze_hz : float or None Temporal analysis rate, in Hz. When ``None``, the BandSILo backend falls back to ``100``. @@ -121,6 +128,7 @@ class SiloParams: peakth: Optional[float] = None activity_channel: int = 0 draw_user_rois: bool = False + interactive: Optional[bool] = None analyze_hz: Optional[float] = None decay_tau_s: float = 0.15 baseline_window_s: float = 4.0 diff --git a/src/giant_python/models/trial_table.py b/src/giant_python/models/trial_table.py index 065b02e..c458d36 100644 --- a/src/giant_python/models/trial_table.py +++ b/src/giant_python/models/trial_table.py @@ -6,6 +6,16 @@ from pathlib import Path from typing import Optional, Union +from ..io import load_struct_h5 + +_SLAP2_FIELDS = ( + "ref_stack", + "first_line", + "last_line", + "trial_start_time_inferred", + "trial_end_time_from_pc", +) + @dataclass class Slap2Info: @@ -76,6 +86,14 @@ class TrialTable: def from_h5(cls, path: Union[str, Path]) -> "TrialTable": """Load a trial table from a MATLAB-compatible trial_table.h5 file. + Faithful deserialization: the on-disk group hierarchy (README + ``trial_table.h5`` schema) is mirrored into the model with no path + resolution or file probing. ``slap2_info`` becomes a + :class:`Slap2Info`; ``motion_correction`` / ``source_extraction`` stay + as nested dicts (present only once those stages have run). Callers that + need a runtime view (absolute ``fn_adata`` paths, kept-trial mask, per- + DMD ``alignHz``) resolve it separately. + Parameters ---------- path : str or Path @@ -86,7 +104,38 @@ def from_h5(cls, path: Union[str, Path]) -> "TrialTable": TrialTable The deserialized trial table. """ - raise NotImplementedError + raw = load_struct_h5(path) + slap2 = raw.get("slap2_info") + return cls( + datadr=Path(raw["datadr"]) if raw.get("datadr") else None, + savedr=Path(raw["savedr"]) if raw.get("savedr") else None, + filename=raw.get("filename"), + true_trial_ix=raw.get("true_trial_ix"), + epoch=raw.get("epoch"), + slap2_info=cls._slap2_info_from_raw(slap2), + motion_correction=raw.get("motion_correction"), + source_extraction=raw.get("source_extraction"), + ) + + @staticmethod + def _slap2_info_from_raw(slap2: Optional[dict]) -> Optional["Slap2Info"]: + """Build a :class:`Slap2Info` from the raw ``slap2_info`` dict. + + Parameters + ---------- + slap2 : dict or None + The faithfully-loaded ``slap2_info`` group (or ``None`` for a + non-SLAP2 experiment). + + Returns + ------- + Slap2Info or None + The populated metadata (only recognized fields), or ``None``. + """ + if not slap2: + return None + kwargs = {k: slap2[k] for k in _SLAP2_FIELDS if k in slap2} + return Slap2Info(**kwargs) def to_h5(self, path: Union[str, Path]) -> None: """Write this trial table to a MATLAB-compatible trial_table.h5 file. diff --git a/src/giant_python/pipeline/pipeline.py b/src/giant_python/pipeline/pipeline.py index fab5625..a5ec2e2 100644 --- a/src/giant_python/pipeline/pipeline.py +++ b/src/giant_python/pipeline/pipeline.py @@ -140,16 +140,27 @@ def register(self) -> "Pipeline": def annotate(self) -> "Pipeline": """Run the optional ROI-annotation GUI step. - Imported lazily so headless installs never import the GUI extra. + Dispatches on the source-extraction scan mode: ``"band"`` uses the + BandSILo standalone annotation step (writing + ``/annotations/annotations.h5``); any other mode uses the + generic annotation GUI. GUI modules are imported lazily so headless + installs never import the GUI extra. Returns ------- Pipeline ``self`` (supports method chaining). """ - from ..gui import annotate_rois + if self.silo_params.scan_mode == "band": + from ..bandsilo.annotate import annotate_band_rois - annotate_rois(self.tt) + annotate_band_rois( + Path(self.tt.savedr) / "trial_table.h5", self.silo_params + ) + else: + from ..gui import annotate_rois + + annotate_rois(self.tt) return self def extract(self) -> "Pipeline": @@ -184,11 +195,16 @@ def run_all( extraction. resume : bool Skip stages whose HDF5 outputs already exist (mirrors the - ``overwriteExisting`` logic in GIAnT-MATLAB). + ``overwriteExisting`` logic in GIAnT-MATLAB). Reserved for the + per-stage checkpoint logic; not yet honored. Returns ------- Pipeline ``self`` (with ``tt`` and ``summary`` populated). """ - raise NotImplementedError + del resume # reserved for future per-stage checkpointing + self.organize(data_dir).register() + if annotate: + self.annotate() + return self.extract() diff --git a/tests/test_bandsilo_annotate.py b/tests/test_bandsilo_annotate.py new file mode 100644 index 0000000..02bab86 --- /dev/null +++ b/tests/test_bandsilo_annotate.py @@ -0,0 +1,151 @@ +"""Tests for giant_python.bandsilo.annotate (pure seams + load/fail-fast).""" + +import tempfile +import unittest + +import numpy as np + +from giant_python.bandsilo import annotate as an +from giant_python.bandsilo.gui import save_annotations_h5 + + +class TestFirstValidTrial(unittest.TestCase): + """first_valid_trial returns the first kept trial or raises.""" + + def test_first_kept(self): + """Returns the index of the first True entry for the DMD.""" + keep = np.array([[False, True, True], [True, False, False]]) + self.assertEqual(an.first_valid_trial(keep, 0), 1) + self.assertEqual(an.first_valid_trial(keep, 1), 0) + + def test_none_kept_raises(self): + """A DMD with no kept trials raises ValueError.""" + keep = np.array([[False, False]]) + with self.assertRaises(ValueError): + an.first_valid_trial(keep, 0) + + +class TestMotionMedian(unittest.TestCase): + """motion_median_from_adata returns per-axis rounded medians.""" + + def test_medians(self): + """Each axis is rounded then reduced by the median.""" + a_data = { + "motionDSr": np.array([1.2, 1.8, 3.1]), + "motionDSc": np.array([0.0, 0.0, 4.0]), + "motionDSz": np.array([5.0, 5.0, 5.0]), + } + self.assertEqual(an.motion_median_from_adata(a_data), (2, 0, 5)) + + +class TestResolveInteractivity(unittest.TestCase): + """resolve_interactivity honors override, env, then TTY fallback.""" + + def test_override_wins(self): + """An explicit override short-circuits detection.""" + self.assertTrue(an.resolve_interactivity(True)) + self.assertFalse(an.resolve_interactivity(False)) + + def test_headless_env(self): + """GIANT_HEADLESS forces headless when no override is given.""" + import os + from unittest import mock + + with mock.patch.dict(os.environ, {"GIANT_HEADLESS": "1"}): + self.assertFalse(an.resolve_interactivity(None)) + + def test_tty_fallback(self): + """With no override/env, falls back to the stdin-TTY probe.""" + import os + from unittest import mock + + with mock.patch.dict(os.environ, {}, clear=True): + with mock.patch.object(an, "_stdin_is_tty", return_value=True): + self.assertTrue(an.resolve_interactivity(None)) + with mock.patch.object(an, "_stdin_is_tty", return_value=False): + self.assertFalse(an.resolve_interactivity(None)) + + +class TestEmptyUserRois(unittest.TestCase): + """_empty_user_rois builds a neutral per-DMD selection.""" + + def test_shape(self): + """Every per-DMD field is an empty list and annotated is False.""" + out = an._empty_user_rois(2) + self.assertFalse(out["annotated"]) + for key in ("DMD1", "DMD2"): + self.assertEqual(out["user_roi_masks"][key], []) + self.assertEqual(out["user_roi_superpixels"][key], []) + self.assertEqual(out["user_roi_labels"][key], []) + self.assertEqual(out["roi_records"][key], []) + + +class TestResolveUserRois(unittest.TestCase): + """resolve_user_rois loads an existing file or fails fast when headless.""" + + def _geo(self): + """Return a minimal single-DMD geometry dict.""" + return { + "DMD1": { + "num_fast_z": 2, + "yx_shape": (4, 5), + "sp_fastz": np.array([0]), + "sp_rows": np.array([1]), + "sp_cols": np.array([1]), + } + } + + def _write_annotations(self, dr): + """Write a one-ROI annotations.h5 into ``dr`` and return its path.""" + mask = np.zeros((2, 4, 5), dtype=bool) + mask[0, 1, 1] = True + records = { + "DMD1": [ + { + "type": "polygon", + "label": "user_roi_1", + "position": np.array( + [[1, 1], [1, 3], [3, 3], [3, 1]], dtype=np.float64 + ), + } + ] + } + return save_annotations_h5(dr, records, {"DMD1": [mask]}, 1) + + def test_loads_existing(self): + """An existing annotations.h5 is loaded even when non-interactive.""" + with tempfile.TemporaryDirectory() as d: + self._write_annotations(d) + out = an.resolve_user_rois(d, 1, self._geo(), interactive=False) + self.assertTrue(out["annotated"]) + self.assertEqual(out["user_roi_labels"]["DMD1"], ["user_roi_1"]) + self.assertTrue(out["user_roi_masks"]["DMD1"][0][0, 1, 1]) + np.testing.assert_array_equal( + out["user_roi_superpixels"]["DMD1"][0], np.array([0]) + ) + + def test_missing_headless_raises(self): + """A missing file + headless raises with actionable guidance.""" + with tempfile.TemporaryDirectory() as d: + with self.assertRaises(RuntimeError) as ctx: + an.resolve_user_rois(d, 1, self._geo(), interactive=False) + self.assertIn("giant annotate", str(ctx.exception)) + + def test_missing_interactive_draws(self): + """A missing file + interactive delegates to the drawing helper.""" + from unittest import mock + + sentinel = {"annotated": True} + with tempfile.TemporaryDirectory() as d: + with mock.patch.object( + an, "_draw_and_save_user_rois", return_value=sentinel + ) as draw: + out = an.resolve_user_rois( + d, 1, self._geo(), interactive=True, ref_files={} + ) + draw.assert_called_once() + self.assertIs(out, sentinel) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_bandsilo_hdf5.py b/tests/test_bandsilo_hdf5.py index 74958d6..286f075 100644 --- a/tests/test_bandsilo_hdf5.py +++ b/tests/test_bandsilo_hdf5.py @@ -12,9 +12,11 @@ _decode_one, _reshape_1d, _resolve_fn_adata, + compute_keep_trials, load_alignment_data_h5, load_struct_from_h5, load_trial_table, + read_align_info, to_serializable, write_dict_to_h5group, ) @@ -196,7 +198,7 @@ def test_load_alignment_data(self): class TestTrialTable(unittest.TestCase): - """_resolve_fn_adata and load_trial_table normalization.""" + """load_trial_table resolve step + keep-trial / align-info helpers.""" def test_resolve_fn_adata(self): """Empty entries stay empty; others become absolute paths.""" @@ -205,14 +207,14 @@ def test_resolve_fn_adata(self): self.assertEqual(out[0, 0], "") self.assertEqual(out[0, 1], os.path.join("root", "moco", "a.h5")) - def _write_trial_table(self, path): + def _write_trial_table(self, path, savedr="/res", with_fn_adata=True): """Write a minimal synthetic trial_table.h5 (row-major).""" fnames = np.array([["t1.dat", "t2.dat"]], dtype=object) fn_adata = np.array([["t1_AD.h5", ""]], dtype=object) with h5py.File(path, "w") as f: f.create_dataset("row_major", data=1) f.create_dataset("datadr", data="/data", dtype=_str_dt()) - f.create_dataset("savedr", data="/res", dtype=_str_dt()) + f.create_dataset("savedr", data=savedr, dtype=_str_dt()) f.create_dataset("filename", data=fnames, dtype=_str_dt()) si = f.create_group("slap2_info") si.create_dataset("first_line", data=np.array([[1, 2]])) @@ -221,32 +223,83 @@ def _write_trial_table(self, path): p1 = rs.create_group("Path1") p1.create_dataset("channels", data=np.array([1, 2])) mc = f.create_group("motion_correction") - mc.create_dataset("fn_adata", data=fn_adata, dtype=_str_dt()) + if with_fn_adata: + mc.create_dataset("fn_adata", data=fn_adata, dtype=_str_dt()) ap = mc.create_group("align_params") ap.create_dataset("align_hz", data=80.0) def test_load_trial_table(self): - """Normalized dict has resolved paths, dims, and align params.""" + """Resolved dict is flat with hoisted arrays and absolute paths.""" with tempfile.TemporaryDirectory() as d: path = os.path.join(d, "trial_table.h5") self._write_trial_table(path) out = load_trial_table(path) - self.assertEqual(out["datadr"], "/data") - self.assertEqual(out["result_dr"], "/res") self.assertEqual( - out["moco_save_dr"], os.path.join("/res", "motion_correction") + os.path.normpath(out["datadr"]), os.path.normpath("/data") + ) + self.assertEqual( + os.path.normpath(out["savedr"]), os.path.normpath("/res") + ) + self.assertEqual( + os.path.normpath(out["moco_save_dr"]), + os.path.normpath(os.path.join("/res", "motion_correction")), ) self.assertEqual(out["n_dmds"], 1) self.assertEqual(out["n_trials"], 2) self.assertAlmostEqual(out["align_params"]["align_hz"], 80.0) - tt = out["trial_table"] - self.assertEqual(tt["fn_adata"][0, 1], "") + self.assertEqual(out["fn_adata"][0, 1], "") self.assertEqual( - tt["fn_adata"][0, 0], - os.path.join("/res", "motion_correction", "t1_AD.h5"), + os.path.normpath(out["fn_adata"][0, 0]), + os.path.normpath( + os.path.join("/res", "motion_correction", "t1_AD.h5") + ), ) - self.assertIn("Path1", tt["ref_stack"]) + self.assertIn("Path1", out["ref_stack"]) + np.testing.assert_array_equal(out["first_line"], np.array([[1, 2]])) + + def test_load_trial_table_unregistered(self): + """A table without motion_correction/fn_adata is rejected.""" + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "trial_table.h5") + self._write_trial_table(path, with_fn_adata=False) + with self.assertRaises(ValueError): + load_trial_table(path) + + def test_compute_keep_trials(self): + """Only trials whose aData and source files both exist are kept.""" + with tempfile.TemporaryDirectory() as d: + adata = os.path.join(d, "t1_AD.h5") + with open(adata, "w") as fh: + fh.write("x") + with open(os.path.join(d, "t1.dat"), "w") as fh: + fh.write("x") + fn_adata = np.array([[adata, ""]], dtype=object) + filename = np.array([["t1.dat", "t2.dat"]], dtype=object) + keep = compute_keep_trials(fn_adata, filename, d) + np.testing.assert_array_equal(keep, np.array([[True, False]])) + + def test_read_align_info(self): + """align_hz/num_channels come from the first kept trial's aData.""" + with tempfile.TemporaryDirectory() as d: + adata = os.path.join(d, "t1_AD.h5") + with h5py.File(adata, "w") as f: + f.create_dataset("row_major", data=1) + f.create_dataset("numChannels", data=2) + f.create_dataset("alignHz", data=80.0) + fn_adata = np.array([[adata, ""]], dtype=object) + keep = np.array([[True, False]]) + align_hz, num_channels = read_align_info(fn_adata, keep, 1) + self.assertAlmostEqual(align_hz["DMD1"], 80.0) + self.assertEqual(num_channels, 2) + + def test_read_align_info_no_valid(self): + """A DMD with no kept trials contributes no align_hz entry.""" + fn_adata = np.array([["", ""]], dtype=object) + keep = np.array([[False, False]]) + align_hz, num_channels = read_align_info(fn_adata, keep, 1) + self.assertEqual(align_hz, {}) + self.assertIsNone(num_channels) if __name__ == "__main__": diff --git a/tests/test_bandsilo_pipeline.py b/tests/test_bandsilo_pipeline.py index 7ea152f..47326f3 100644 --- a/tests/test_bandsilo_pipeline.py +++ b/tests/test_bandsilo_pipeline.py @@ -77,6 +77,39 @@ def _path_result( ) +class TestDmdUserRois(unittest.TestCase): + """_dmd_user_rois slices a resolved session selection per DMD.""" + + def test_none_is_neutral(self): + """No session selection yields (None, None, []).""" + self.assertEqual(pl._dmd_user_rois(None, "DMD1"), (None, None, [])) + + def test_slices_key(self): + """The named DMD's masks, labels, and superpixels are returned.""" + sps = [np.array([0, 1])] + user_rois = { + "user_roi_masks": {"DMD1": ["mask"]}, + "user_roi_labels": {"DMD1": ["roi"]}, + "user_roi_superpixels": {"DMD1": sps}, + } + masks, labels, out_sps = pl._dmd_user_rois(user_rois, "DMD1") + self.assertEqual(masks, ["mask"]) + self.assertEqual(labels, ["roi"]) + self.assertIs(out_sps, sps) + + def test_missing_key_defaults(self): + """A DMD absent from the selection yields empty superpixels.""" + user_rois = { + "user_roi_masks": {}, + "user_roi_labels": {}, + "user_roi_superpixels": {}, + } + masks, labels, out_sps = pl._dmd_user_rois(user_rois, "DMD2") + self.assertIsNone(masks) + self.assertIsNone(labels) + self.assertEqual(out_sps, []) + + class TestWithNanChannel(unittest.TestCase): """_with_nan_channel reshapes and NaN-fills non-activity channels.""" diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..1d82e3f --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,99 @@ +"""Tests for the giant_python.cli argument parser and dispatch.""" + +import unittest +from unittest import mock + +from giant_python import cli + + +class TestBuildParser(unittest.TestCase): + """build_parser wires the four subcommands and their options.""" + + def setUp(self): + """Build a fresh parser for each test.""" + self.parser = cli.build_parser() + + def test_no_command(self): + """With no subcommand, command is None.""" + args = self.parser.parse_args([]) + self.assertIsNone(args.command) + + def test_organize(self): + """organize captures the data and save dirs.""" + args = self.parser.parse_args(["organize", "/data", "/results"]) + self.assertEqual(args.command, "organize") + self.assertEqual(args.data_dir, "/data") + self.assertEqual(args.save_dir, "/results") + + def test_annotate_defaults(self): + """annotate defaults to band scan, no draw, auto interactivity.""" + args = self.parser.parse_args(["annotate", "tt.h5"]) + self.assertEqual(args.command, "annotate") + self.assertEqual(args.trial_table, "tt.h5") + self.assertEqual(args.scan_mode, "band") + self.assertFalse(args.draw_user_rois) + self.assertIsNone(args.interactive) + + def test_extract_flags(self): + """extract parses the draw and headless flags.""" + args = self.parser.parse_args( + ["extract", "tt.h5", "--draw-user-rois", "--headless"] + ) + self.assertTrue(args.draw_user_rois) + self.assertFalse(args.interactive) + + def test_interactive_flag(self): + """--interactive resolves to True.""" + args = self.parser.parse_args(["annotate", "tt.h5", "--interactive"]) + self.assertTrue(args.interactive) + + +class TestParamsFromArgs(unittest.TestCase): + """_params_from_args builds a SiloParams from the namespace.""" + + def test_maps_fields(self): + """Each parsed field maps onto the SiloParams.""" + args = cli.build_parser().parse_args( + [ + "extract", + "tt.h5", + "--scan-mode", + "band", + "--draw-user-rois", + "--interactive", + "--operator", + "Alice", + ] + ) + params = cli._params_from_args(args) + self.assertEqual(params.scan_mode, "band") + self.assertTrue(params.draw_user_rois) + self.assertTrue(params.interactive) + self.assertEqual(params.operator, "Alice") + + +class TestMain(unittest.TestCase): + """main dispatches to handlers and handles the no-command case.""" + + def test_no_command_prints_help(self): + """No subcommand prints help and returns 2.""" + parser = cli.build_parser() + with mock.patch.object( + cli, "build_parser", return_value=parser + ), mock.patch.object(parser, "print_help") as help_: + code = cli.main([]) + help_.assert_called_once() + self.assertEqual(code, 2) + + def test_dispatches_annotate(self): + """main routes the annotate command to its handler.""" + with mock.patch.object( + cli, "_cmd_annotate", return_value=0 + ) as handler: + code = cli.main(["annotate", "tt.h5"]) + handler.assert_called_once() + self.assertEqual(code, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_io.py b/tests/test_io.py index 8c537e6..18d6ad2 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -1,7 +1,10 @@ """Tests for the io subpackage.""" +import os +import tempfile import unittest +import h5py import numpy as np from giant_python.io import ( @@ -13,15 +16,60 @@ scanimagetiff_data_wrapper, scanimagetiff_wrapper, ) +from giant_python.io.hdf5 import _decode_one, _decode_strings + + +class TestDecodeHelpers(unittest.TestCase): + """_decode_one / _decode_strings handle bytes, str, and empty arrays.""" + + def test_decode_one(self): + """bytes decode via utf-8; non-bytes pass through str().""" + self.assertEqual(_decode_one(b"ab"), "ab") + self.assertEqual(_decode_one("cd"), "cd") + + def test_decode_empty_array(self): + """An empty array is returned unchanged (no vectorize crash).""" + out = _decode_strings(np.array([], dtype=object)) + self.assertEqual(out.size, 0) class TestHdf5(unittest.TestCase): """Tests for the generic struct <-> HDF5 helpers.""" - def test_load_struct_h5(self): - """load_struct_h5 raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - load_struct_h5("dummy.h5") + def test_load_struct_h5_faithful(self): + """load_struct_h5 mirrors groups, decodes strings, honors row_major.""" + str_dt = h5py.string_dtype(encoding="utf-8") + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "s.h5") + with h5py.File(path, "w") as f: + # Column-major (row_major absent) -> 2-D axes transposed. + f.create_dataset("name", data="hello", dtype=str_dt) + f.create_dataset("mat", data=np.array([[1, 2, 3], [4, 5, 6]])) + grp = f.create_group("inner") + grp.create_dataset( + "labels", + data=np.array([["a", "b"]], dtype=object), + dtype=str_dt, + ) + out = load_struct_h5(path) + self.assertEqual(out["name"], "hello") + # (2, 3) written column-major reads back transposed to (3, 2). + np.testing.assert_array_equal( + out["mat"], np.array([[1, 4], [2, 5], [3, 6]]) + ) + np.testing.assert_array_equal( + out["inner"]["labels"], np.array([["a"], ["b"]], dtype=object) + ) + + def test_load_struct_h5_row_major(self): + """A row_major file keeps its on-disk orientation.""" + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "s.h5") + with h5py.File(path, "w") as f: + f.create_dataset("row_major", data=1) + f.create_dataset("mat", data=np.array([[1, 2, 3]])) + out = load_struct_h5(path) + np.testing.assert_array_equal(out["mat"], np.array([[1, 2, 3]])) def test_save_struct_h5(self): """save_struct_h5 raises NotImplementedError.""" diff --git a/tests/test_models.py b/tests/test_models.py index 0a00881..fbda2ea 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,7 +1,12 @@ """Tests for the models subpackage.""" +import os +import tempfile import unittest +import h5py +import numpy as np + from giant_python.models import ( AlignmentData, AlignParams, @@ -55,9 +60,60 @@ class TestH5RoundTrip(unittest.TestCase): """from_h5/to_h5 are not implemented yet.""" def test_trial_table_from_h5(self): - """TrialTable.from_h5 raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - TrialTable.from_h5("trial_table.h5") + """from_h5 faithfully mirrors the trial_table.h5 group hierarchy.""" + str_dt = h5py.string_dtype(encoding="utf-8") + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "trial_table.h5") + with h5py.File(path, "w") as f: + f.create_dataset("row_major", data=1) + f.create_dataset("datadr", data="/data", dtype=str_dt) + f.create_dataset("savedr", data="/res", dtype=str_dt) + f.create_dataset( + "filename", + data=np.array([["t1", "t2"]], dtype=object), + dtype=str_dt, + ) + si = f.create_group("slap2_info") + si.create_dataset("first_line", data=np.array([[1, 2]])) + si.create_dataset("last_line", data=np.array([[10, 20]])) + rs = si.create_group("ref_stack") + rs.create_group("Path1").create_dataset( + "channels", data=np.array([1, 2]) + ) + mc = f.create_group("motion_correction") + mc.create_dataset( + "fn_adata", + data=np.array([["a.h5", ""]], dtype=object), + dtype=str_dt, + ) + tt = TrialTable.from_h5(path) + self.assertEqual(str(tt.datadr), os.path.normpath("/data")) + self.assertEqual(str(tt.savedr), os.path.normpath("/res")) + self.assertIsInstance(tt.slap2_info, Slap2Info) + np.testing.assert_array_equal( + np.asarray(tt.slap2_info.first_line), np.array([[1, 2]]) + ) + self.assertIn("Path1", tt.slap2_info.ref_stack) + self.assertEqual(tt.motion_correction["fn_adata"][0, 0], "a.h5") + self.assertIsNone(tt.source_extraction) + + def test_trial_table_from_h5_non_slap2(self): + """A table without slap2_info yields slap2_info=None.""" + str_dt = h5py.string_dtype(encoding="utf-8") + with tempfile.TemporaryDirectory() as d: + path = os.path.join(d, "trial_table.h5") + with h5py.File(path, "w") as f: + f.create_dataset("row_major", data=1) + f.create_dataset("datadr", data="/data", dtype=str_dt) + f.create_dataset("savedr", data="/res", dtype=str_dt) + f.create_dataset( + "filename", + data=np.array([["t1"]], dtype=object), + dtype=str_dt, + ) + tt = TrialTable.from_h5(path) + self.assertIsNone(tt.slap2_info) + self.assertIsNone(tt.motion_correction) def test_trial_table_to_h5(self): """TrialTable.to_h5 raises NotImplementedError.""" diff --git a/tests/test_parallel.py b/tests/test_parallel.py index 0992917..fa73833 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -1,8 +1,7 @@ -"""Tests for the parallel helper and the CLI entry point.""" +"""Tests for the per-trial parallelism helper.""" import unittest -from giant_python.cli import main from giant_python.parallel import map_trials @@ -31,14 +30,5 @@ def test_parallel(self): ) -class TestCli(unittest.TestCase): - """Tests for the CLI entry point.""" - - def test_not_implemented(self): - """main raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - main([]) - - if __name__ == "__main__": unittest.main() diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 7d6da72..0576c9d 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -182,12 +182,26 @@ def test_register_delegates(self): self.assertEqual(self.pipe.tt, "TT2") def test_annotate_delegates(self): - """annotate lazily invokes the ROI-annotation GUI step.""" + """A non-band scan mode uses the generic annotation GUI.""" + self.pipe.silo_params.scan_mode = "standard" with mock.patch("giant_python.gui.annotate_rois") as annotate_rois: result = self.pipe.annotate() annotate_rois.assert_called_once() self.assertIs(result, self.pipe) + def test_annotate_band_delegates(self): + """A band scan mode uses the BandSILo standalone annotation step.""" + self.pipe.silo_params.scan_mode = "band" + self.pipe.tt = mock.Mock(savedr="/results") + with mock.patch( + "giant_python.bandsilo.annotate.annotate_band_rois" + ) as annotate_band: + result = self.pipe.annotate() + annotate_band.assert_called_once() + called_path = annotate_band.call_args[0][0] + self.assertEqual(called_path.name, "trial_table.h5") + self.assertIs(result, self.pipe) + def test_extract_delegates(self): """extract dispatches by scan mode and stores the summary.""" self.pipe.silo_params.scan_mode = "band" @@ -199,10 +213,37 @@ def test_extract_delegates(self): self.assertIs(result, self.pipe) self.assertEqual(self.pipe.summary, "SUMMARY") - def test_run_all(self): - """run_all raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): + def test_run_all_sequences_stages(self): + """run_all sequences organize -> register -> annotate -> extract.""" + with mock.patch.object( + self.pipe, "organize", return_value=self.pipe + ) as organize, mock.patch.object( + self.pipe, "register", return_value=self.pipe + ) as register, mock.patch.object( + self.pipe, "annotate", return_value=self.pipe + ) as annotate, mock.patch.object( + self.pipe, "extract", return_value=self.pipe + ) as extract: + result = self.pipe.run_all("/data", annotate=True) + organize.assert_called_once_with("/data") + register.assert_called_once() + annotate.assert_called_once() + extract.assert_called_once() + self.assertIs(result, self.pipe) + + def test_run_all_skips_annotation(self): + """run_all skips annotation when annotate is False.""" + with mock.patch.object( + self.pipe, "organize", return_value=self.pipe + ), mock.patch.object( + self.pipe, "register", return_value=self.pipe + ), mock.patch.object( + self.pipe, "annotate", return_value=self.pipe + ) as annotate, mock.patch.object( + self.pipe, "extract", return_value=self.pipe + ): self.pipe.run_all("/data") + annotate.assert_not_called() def test_from_trial_table(self): """from_trial_table raises NotImplementedError.""" From da7e7767bc103bdc8a59463fcd0e9a63845cca6d Mon Sep 17 00:00:00 2001 From: Michael Xie Date: Thu, 6 Aug 2026 20:02:57 -0700 Subject: [PATCH 13/55] add verbose mode --- src/giant_python/bandsilo/annotate.py | 11 +++- src/giant_python/bandsilo/background.py | 23 +++++++- src/giant_python/bandsilo/nmf.py | 11 +++- src/giant_python/bandsilo/pipeline.py | 44 ++++++++++++-- src/giant_python/bandsilo/progress.py | 63 +++++++++++++++++++++ src/giant_python/bandsilo/summary_images.py | 8 ++- src/giant_python/cli.py | 6 ++ src/giant_python/models/params.py | 6 ++ src/giant_python/parallel.py | 21 ++++++- tests/test_bandsilo_progress.py | 44 ++++++++++++++ tests/test_cli.py | 3 + tests/test_parallel.py | 6 ++ 12 files changed, 231 insertions(+), 15 deletions(-) create mode 100644 src/giant_python/bandsilo/progress.py create mode 100644 tests/test_bandsilo_progress.py diff --git a/src/giant_python/bandsilo/annotate.py b/src/giant_python/bandsilo/annotate.py index ddb3feb..1b988a4 100644 --- a/src/giant_python/bandsilo/annotate.py +++ b/src/giant_python/bandsilo/annotate.py @@ -38,6 +38,7 @@ save_annotations_h5, ) from .hdf5 import compute_keep_trials, load_alignment_data_h5, load_trial_table +from .progress import log def first_valid_trial(keep_trials: np.ndarray, dmd_ix: int) -> int: @@ -307,6 +308,7 @@ def annotate_band_rois( Path to the written (or existing) ``annotations.h5``. """ params = _resolve_params(params_in) + log(f"Loading trial table {path_to_trial_table}", params.verbose) trial_table = load_trial_table(path_to_trial_table) trial_table["keep_trials"] = compute_keep_trials( trial_table["fn_adata"], @@ -319,6 +321,11 @@ def annotate_band_rois( Path(trial_table["moco_save_dr"]) / "bandRegLookupTable.h5", trial_table["n_dmds"], ) + log( + f"Loading reference geometry for {trial_table['n_dmds']} " + "DMD path(s)", + params.verbose, + ) user_roi_geo, ref_files = build_user_roi_geometry(trial_table, lookup) resolve_user_rois( @@ -328,4 +335,6 @@ def annotate_band_rois( interactive=resolve_interactivity(params.interactive), ref_files=ref_files, ) - return os.path.join(str(annotation_dir), "annotations.h5") + out_path = os.path.join(str(annotation_dir), "annotations.h5") + log(f"Annotations at {out_path}", params.verbose) + return out_path diff --git a/src/giant_python/bandsilo/background.py b/src/giant_python/bandsilo/background.py index b258f7a..4813c11 100644 --- a/src/giant_python/bandsilo/background.py +++ b/src/giant_python/bandsilo/background.py @@ -33,6 +33,8 @@ from scipy import signal from scipy.interpolate import RectBivariateSpline +from .progress import progress + def bin_motion( motion_r: np.ndarray, motion_c: np.ndarray, motion_z: np.ndarray @@ -904,6 +906,7 @@ def compute_rho( dmd_pixels_per_column: int, dmd_pixels_per_row: int, psf2d: np.ndarray, + verbose: bool = False, ) -> np.ndarray: """Compute the ``rho`` matched-filter response on the selected pixels. @@ -932,6 +935,8 @@ def compute_rho( Grid geometry. psf2d : ndarray The (cropped) PSF for this DMD. + verbose : bool + Show a per-motion-bin progress bar when set. Returns ------- @@ -948,7 +953,11 @@ def compute_rho( (len(sel_pix_idxs), residual.shape[1]), np.nan, dtype=np.float32 ) - for i in range(len(unique_motion_to_keep_yx)): + for i in progress( + range(len(unique_motion_to_keep_yx)), + desc="Computing rho", + verbose=verbose, + ): motion_frames = np.flatnonzero(mot_inds_yx == i) if motion_frames.size == 0: continue @@ -1066,7 +1075,9 @@ def decay_kernel_1d(decay_tau_s: float, align_hz: float) -> np.ndarray: return k1d / np.sum(k1d) -def smooth_rho(rho: np.ndarray, k1d: np.ndarray) -> np.ndarray: +def smooth_rho( + rho: np.ndarray, k1d: np.ndarray, verbose: bool = False +) -> np.ndarray: """NaN-aware temporal smoothing of ``rho`` with a 1-D decay kernel. Convolves along time only, dividing the smoothed values by the smoothed @@ -1080,6 +1091,8 @@ def smooth_rho(rho: np.ndarray, k1d: np.ndarray) -> np.ndarray: The rho response (NaN allowed). k1d : ndarray 1-D smoothing kernel from :func:`decay_kernel_1d`. + verbose : bool + Show a per-row-chunk progress bar when set. Returns ------- @@ -1091,7 +1104,11 @@ def smooth_rho(rho: np.ndarray, k1d: np.ndarray) -> np.ndarray: bytes_per_row = max(1, n_time) * np.dtype(np.float32).itemsize * 3 row_chunk = max(64, min(4096, int(200_000_000 // bytes_per_row))) - for r0 in range(0, rho.shape[0], row_chunk): + for r0 in progress( + range(0, rho.shape[0], row_chunk), + desc="Smoothing rho", + verbose=verbose, + ): r1 = min(r0 + row_chunk, rho.shape[0]) rc = rho[r0:r1].copy() row_has_data = np.any(np.isfinite(rc), axis=1) diff --git a/src/giant_python/bandsilo/nmf.py b/src/giant_python/bandsilo/nmf.py index 0c6f655..d4d05b5 100644 --- a/src/giant_python/bandsilo/nmf.py +++ b/src/giant_python/bandsilo/nmf.py @@ -37,6 +37,8 @@ import numpy as np import torch +from .progress import progress + def sel_pix_gaussian_profile( gaussian_params: torch.Tensor, pixel_coords_tensor: torch.Tensor @@ -654,6 +656,7 @@ def fit_sources( mult_nmf_max_iters: int = 10, learning_rate: float = 0.01, gd_tol: float = 1e-4, + verbose: bool = False, ) -> dict: """Localize sources by superpixel-space NMF + Gaussian-profile fitting. @@ -687,6 +690,8 @@ def fit_sources( Base Adam learning rate. gd_tol : float Adam convergence tolerance. + verbose : bool + Show a progress bar over the outer NMF/fit iterations when set. Returns ------- @@ -716,7 +721,11 @@ def fit_sources( (n_frames, n_sources), float("nan"), dtype=torch.float32 ) - for outer_loop_iter in range(outer_loop_iters): + for outer_loop_iter in progress( + range(outer_loop_iters), + desc="Localizing sources (NMF)", + verbose=verbose, + ): x_mots = multiplicative_nmf( a, h_mots, diff --git a/src/giant_python/bandsilo/pipeline.py b/src/giant_python/bandsilo/pipeline.py index 9b936f0..091d3e6 100644 --- a/src/giant_python/bandsilo/pipeline.py +++ b/src/giant_python/bandsilo/pipeline.py @@ -53,6 +53,7 @@ ) from .nmf import fit_sources from .peaks import get_act_im_peaks +from .progress import log, progress from .traces import get_high_res_traces _FRAME_INFO_KEYS = ( @@ -509,6 +510,7 @@ def extract_band_sources( The extracted sources and summary (also written to disk). """ params = _resolve_params(params_in) + log(f"Loading trial table {path_to_trial_table}", params.verbose) trial_table = load_trial_table(path_to_trial_table) result_dr = str(trial_table["savedr"]) src_extr_dr = Path(result_dr) / "source_extraction" @@ -537,8 +539,10 @@ def extract_band_sources( result_dr, trial_table, lookup, params ) + n_dmds = trial_table["n_dmds"] + log(f"Extracting sources for {n_dmds} DMD path(s)", params.verbose) assembled_paths = [] - for dmd_ix in range(trial_table["n_dmds"]): + for dmd_ix in range(n_dmds): assembled_paths.append( assemble_path_outputs( _process_dmd( @@ -548,6 +552,7 @@ def extract_band_sources( ) params_serializable = to_serializable(_params_dict(params)) + log(f"Writing {output_path}", params.verbose) write_experiment_summary(output_path, params_serializable, assembled_paths) return build_experiment_summary(params_serializable, assembled_paths) @@ -588,6 +593,7 @@ def _process_dmd( the per-trial ``F_soma``. """ key = f"DMD{dmd_ix + 1}" + log(f"Processing {key}", params.verbose) soma_masks, soma_labels, soma_sps = _dmd_user_rois(user_rois, key) datadr = str(trial_table["datadr"]) n_trials = trial_table["filename"].shape[1] @@ -632,7 +638,13 @@ def _read(trial_ix): ) low = td.assemble_lowres_data( - map_trials(_read, range(n_trials), params.max_workers), num_channels + map_trials( + _read, + range(n_trials), + params.max_workers, + desc=f"{key} reading trials" if params.verbose else None, + ), + num_channels, ) low_res_data_norm = low["lowResData"] / low["lowResDataCt"] v_im = 1.0 / low["lowResDataCt"] @@ -751,7 +763,12 @@ def _traces(trial_ix): soma_sps, ) - trace_results = map_trials(_traces, range(n_trials), params.max_workers) + trace_results = map_trials( + _traces, + range(n_trials), + params.max_workers, + desc=f"{key} extracting traces" if params.verbose else None, + ) trace_results = [r for r in trace_results if r[0].shape[0] > 0] return PathResult( @@ -797,13 +814,16 @@ def _estimate_background( params, ): # pragma: no cover - Phase-4 background composition over real data """Interpolated + rolling-baseline background on the superpixel grid.""" + log("Estimating background", params.verbose) psf_tensor, psf_center, psf_exp, psf_center_exp = bg.expand_psf(psf2d) interp_data = np.full( (sel_pix_idxs.shape[0], low_res_data_norm.shape[1]), np.nan, dtype=np.float32, ) - for z in range(num_fast_zs): + for z in progress( + range(num_fast_zs), desc="Interpolating planes", verbose=params.verbose + ): z_idxs, sel_2d = bg.selected_pixels_2d_for_plane( sel_pix_idxs, z, dmd_pixels_per_column, dmd_pixels_per_row ) @@ -860,6 +880,7 @@ def _localize( params, ): # pragma: no cover - Phase 4-6 rho/activity/NMF over real data """Compute rho + activity image, detect peaks, and NMF-fit sources.""" + log("Computing activity image and localizing sources", params.verbose) h_mots = bg.build_motion_h_matrices( sparse_h_inds, sparse_h_vals, @@ -870,7 +891,11 @@ def _localize( ) psf_tensor, psf_center, psf_exp, psf_center_exp = bg.expand_psf(psf2d) d_mats, d_mats_exp = [], [] - for z in range(num_fast_zs): + for z in progress( + range(num_fast_zs), + desc="Computing D matrices", + verbose=params.verbose, + ): _, sel_2d = bg.selected_pixels_2d_for_plane( sel_pix_idxs, z, dmd_pixels_per_column, dmd_pixels_per_row ) @@ -895,9 +920,14 @@ def _localize( dmd_pixels_per_column, dmd_pixels_per_row, psf2d, + verbose=params.verbose, ) nan_ct = bg.mask_high_nan_rho(rho) - rho = bg.smooth_rho(rho, bg.decay_kernel_1d(params.decay_tau_s, align_hz)) + rho = bg.smooth_rho( + rho, + bg.decay_kernel_1d(params.decay_tau_s, align_hz), + verbose=params.verbose, + ) act_im = si.accumulate_activity_image( rho, @@ -905,6 +935,7 @@ def _localize( num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row, + verbose=params.verbose, ) act_im = si.finalize_activity_image(act_im, sel_pix_idxs, nan_ct) @@ -925,6 +956,7 @@ def _localize( n_pixels, params.d_xy, params.sparse_fac, + verbose=params.verbose, ) return ( act_im, diff --git a/src/giant_python/bandsilo/progress.py b/src/giant_python/bandsilo/progress.py new file mode 100644 index 0000000..990d6ee --- /dev/null +++ b/src/giant_python/bandsilo/progress.py @@ -0,0 +1,63 @@ +"""Opt-in verbose logging + progress bars for the band pipeline. + +The BandSILo port stays quiet by default (headless / batch friendly); the +reference's unconditional ``print``s and ``tqdm`` bars were dropped. These +helpers restore that feedback opt-in: when ``verbose`` is set +(``SiloParams.verbose`` / ``giant ... --verbose``) stage messages are printed +and long-running loops show a progress bar. ``tqdm`` is imported lazily, so a +non-verbose run neither prints nor imports it. +""" + +from __future__ import annotations + +from typing import Iterable, Optional + + +def log(message: str, verbose: bool) -> None: + """Print ``message`` when ``verbose`` is set; otherwise do nothing. + + Parameters + ---------- + message : str + The status message to display. + verbose : bool + Whether verbose output is enabled. + """ + if verbose: + print(message) + + +def progress( + iterable: Iterable, + *, + desc: str, + verbose: bool, + total: Optional[int] = None, +) -> Iterable: + """Wrap ``iterable`` in a tqdm progress bar when ``verbose`` is set. + + Returns the iterable unchanged when ``verbose`` is ``False`` so callers can + wrap any loop unconditionally at zero cost (and without importing + ``tqdm``). + + Parameters + ---------- + iterable : iterable + The loop iterable to (optionally) wrap. + desc : str + Progress-bar label. + verbose : bool + Whether to show the bar. + total : int, optional + Item count, for iterables without a ``len`` (e.g. a generator). + + Returns + ------- + iterable + The tqdm-wrapped iterable, or the original iterable. + """ + if not verbose: + return iterable + from tqdm import tqdm + + return tqdm(iterable, desc=desc, total=total) diff --git a/src/giant_python/bandsilo/summary_images.py b/src/giant_python/bandsilo/summary_images.py index a163a36..1ad60e7 100644 --- a/src/giant_python/bandsilo/summary_images.py +++ b/src/giant_python/bandsilo/summary_images.py @@ -25,6 +25,7 @@ import numpy as np import scipy.ndimage as ndimage +from .progress import progress from .trial_data import fast_dilation @@ -100,6 +101,7 @@ def accumulate_activity_image( dmd_pixels_per_column: int, dmd_pixels_per_row: int, batch_size: int = 1000, + verbose: bool = False, ) -> np.ndarray: """Accumulate squared spatio-temporal local maxima of ``rho``. @@ -119,6 +121,8 @@ def accumulate_activity_image( Spatial grid geometry. batch_size : int Number of frames processed per batch (capped at ``n_frames``). + verbose : bool + Show a per-batch progress bar when set. Returns ------- @@ -176,7 +180,9 @@ def accumulate_activity_image( batch_rho_pow2_core = np.empty(interior_shape, dtype=np.float32) time_indices_pre = np.arange(prealloc_size)[:, None] - for batch_idx in range(num_batches): + for batch_idx in progress( + range(num_batches), desc="Creating activity map", verbose=verbose + ): batch_start = batch_idx * batch_size batch_end = min(batch_start + batch_size, n_frames) padded_start = max(0, batch_start - temporal_pad) diff --git a/src/giant_python/cli.py b/src/giant_python/cli.py index e739ace..69ea23a 100644 --- a/src/giant_python/cli.py +++ b/src/giant_python/cli.py @@ -52,6 +52,11 @@ def _add_silo_options(parser: argparse.ArgumentParser) -> None: default=SiloParams.operator, help="Operator name recorded in the output metadata.", ) + parser.add_argument( + "--verbose", + action="store_true", + help="Print per-stage status messages and progress bars.", + ) def build_parser() -> argparse.ArgumentParser: @@ -96,6 +101,7 @@ def _params_from_args(args: argparse.Namespace) -> SiloParams: draw_user_rois=args.draw_user_rois, interactive=args.interactive, operator=args.operator, + verbose=args.verbose, ) diff --git a/src/giant_python/models/params.py b/src/giant_python/models/params.py index 5bfd328..579066e 100644 --- a/src/giant_python/models/params.py +++ b/src/giant_python/models/params.py @@ -116,6 +116,11 @@ class SiloParams: Dilation size selecting the bundled ``dil-NN.tif`` PSF template. operator : str Operator name recorded in the output metadata. + verbose : bool + When set, the band pipeline prints per-stage status messages and shows + tqdm progress bars over its long-running loops (per-trial reads, rho, + activity image, NMF). Off by default so headless / batch runs stay + quiet. See :mod:`giant_python.bandsilo.progress`. """ microscope: str = "slap2" @@ -141,6 +146,7 @@ class SiloParams: num_channels: Optional[int] = None psf_dilation: int = 17 operator: str = "SLAP2 User" + verbose: bool = False def set_params( diff --git a/src/giant_python/parallel.py b/src/giant_python/parallel.py index 3bdff5c..082fd12 100644 --- a/src/giant_python/parallel.py +++ b/src/giant_python/parallel.py @@ -5,16 +5,26 @@ trials without each stage reimplementing the plumbing. """ -from typing import Callable, Iterable, List, TypeVar +from typing import Callable, Iterable, List, Optional, TypeVar T = TypeVar("T") R = TypeVar("R") +def _maybe_bar(trials: list, desc: Optional[str]): + """Wrap ``trials`` in a tqdm bar when ``desc`` is set, else as-is.""" + if desc is None: + return trials + from tqdm import tqdm + + return tqdm(trials, desc=desc) + + def map_trials( func: Callable[[T], R], trials: Iterable[T], n_workers: int = 1, + desc: Optional[str] = None, ) -> List[R]: """Apply *func* to each trial, optionally in parallel. @@ -30,6 +40,9 @@ def map_trials( The trials to process. n_workers : int Number of parallel workers. ``1`` runs serially. + desc : str, optional + When set, display a tqdm progress bar with this label as the trials are + dispatched (used by the verbose band pipeline). ``None`` shows nothing. Returns ------- @@ -38,8 +51,10 @@ def map_trials( """ trials = list(trials) if n_workers <= 1 or len(trials) <= 1: - return [func(t) for t in trials] + return [func(t) for t in _maybe_bar(trials, desc)] from joblib import Parallel, delayed - return Parallel(n_jobs=n_workers)(delayed(func)(t) for t in trials) + return Parallel(n_jobs=n_workers)( + delayed(func)(t) for t in _maybe_bar(trials, desc) + ) diff --git a/tests/test_bandsilo_progress.py b/tests/test_bandsilo_progress.py new file mode 100644 index 0000000..3a28b89 --- /dev/null +++ b/tests/test_bandsilo_progress.py @@ -0,0 +1,44 @@ +"""Tests for giant_python.bandsilo.progress (opt-in logging + bars).""" + +import contextlib +import io +import unittest + +from giant_python.bandsilo.progress import log, progress + + +class TestLog(unittest.TestCase): + """log prints only when verbose is set.""" + + def test_verbose_prints(self): + """A verbose log writes the message to stdout.""" + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + log("hello", True) + self.assertEqual(buf.getvalue().strip(), "hello") + + def test_quiet_is_silent(self): + """A non-verbose log writes nothing.""" + buf = io.StringIO() + with contextlib.redirect_stdout(buf): + log("hello", False) + self.assertEqual(buf.getvalue(), "") + + +class TestProgress(unittest.TestCase): + """progress wraps an iterable only when verbose is set.""" + + def test_quiet_returns_same_object(self): + """Without verbose, the iterable is returned unchanged.""" + r = range(3) + self.assertIs(progress(r, desc="x", verbose=False), r) + + def test_verbose_wraps_but_preserves_items(self): + """With verbose, a tqdm wrapper still yields the same items.""" + out = progress(range(3), desc="x", verbose=True) + self.assertIsNot(out, range(3)) + self.assertEqual(list(out), [0, 1, 2]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_cli.py b/tests/test_cli.py index 1d82e3f..5a43065 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -33,6 +33,7 @@ def test_annotate_defaults(self): self.assertEqual(args.scan_mode, "band") self.assertFalse(args.draw_user_rois) self.assertIsNone(args.interactive) + self.assertFalse(args.verbose) def test_extract_flags(self): """extract parses the draw and headless flags.""" @@ -63,6 +64,7 @@ def test_maps_fields(self): "--interactive", "--operator", "Alice", + "--verbose", ] ) params = cli._params_from_args(args) @@ -70,6 +72,7 @@ def test_maps_fields(self): self.assertTrue(params.draw_user_rois) self.assertTrue(params.interactive) self.assertEqual(params.operator, "Alice") + self.assertTrue(params.verbose) class TestMain(unittest.TestCase): diff --git a/tests/test_parallel.py b/tests/test_parallel.py index fa73833..4312163 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -29,6 +29,12 @@ def test_parallel(self): map_trials(_double, [1, 2, 3, 4], n_workers=2), [2, 4, 6, 8] ) + def test_desc_shows_progress_bar(self): + """A desc label wraps the trials in a bar without changing results.""" + self.assertEqual( + map_trials(_double, [1, 2, 3], desc="doubling"), [2, 4, 6] + ) + if __name__ == "__main__": unittest.main() From a7c2fc7acdf45c8dbba6cdff89474b85da1bbe49 Mon Sep 17 00:00:00 2001 From: Michael Xie Date: Fri, 7 Aug 2026 11:02:13 -0700 Subject: [PATCH 14/55] set default analyze_hz --- src/giant_python/bandsilo/pipeline.py | 22 ++++++++++++++++++++++ tests/test_bandsilo_pipeline.py | 14 ++++++++++++++ 2 files changed, 36 insertions(+) diff --git a/src/giant_python/bandsilo/pipeline.py b/src/giant_python/bandsilo/pipeline.py index 091d3e6..5ea14fe 100644 --- a/src/giant_python/bandsilo/pipeline.py +++ b/src/giant_python/bandsilo/pipeline.py @@ -437,6 +437,27 @@ def _resolve_params( return SiloParams(**params_in) +def effective_analyze_hz(analyze_hz) -> float: + """Return the band analysis rate (Hz), defaulting to 100 when unset. + + ``SiloParams.analyze_hz`` defaults to ``None`` (the reference's parameter + GUI always supplied a value, defaulting to 100 Hz). The band backend + materializes that fallback here so it is used consistently as the + high-res-trace sample rate and in the dF/F window sizes. + + Parameters + ---------- + analyze_hz : float or None + The configured analysis rate. + + Returns + ------- + float + ``analyze_hz`` as a float, or ``100.0`` when it is ``None``. + """ + return 100.0 if analyze_hz is None else float(analyze_hz) + + def _resolve_session_user_rois( result_dr: str, trial_table: dict, @@ -510,6 +531,7 @@ def extract_band_sources( The extracted sources and summary (also written to disk). """ params = _resolve_params(params_in) + params.analyze_hz = effective_analyze_hz(params.analyze_hz) log(f"Loading trial table {path_to_trial_table}", params.verbose) trial_table = load_trial_table(path_to_trial_table) result_dr = str(trial_table["savedr"]) diff --git a/tests/test_bandsilo_pipeline.py b/tests/test_bandsilo_pipeline.py index 47326f3..8513c33 100644 --- a/tests/test_bandsilo_pipeline.py +++ b/tests/test_bandsilo_pipeline.py @@ -77,6 +77,20 @@ def _path_result( ) +class TestEffectiveAnalyzeHz(unittest.TestCase): + """effective_analyze_hz applies the 100 Hz default for None.""" + + def test_none_defaults_to_100(self): + """An unset analyze_hz falls back to 100.0.""" + self.assertEqual(pl.effective_analyze_hz(None), 100.0) + + def test_value_passthrough(self): + """A set analyze_hz is returned as a float.""" + out = pl.effective_analyze_hz(80) + self.assertEqual(out, 80.0) + self.assertIsInstance(out, float) + + class TestDmdUserRois(unittest.TestCase): """_dmd_user_rois slices a resolved session selection per DMD.""" From 69dda2c7318953fa665a0eb5e6421b93dd142553 Mon Sep 17 00:00:00 2001 From: Michael Xie Date: Fri, 7 Aug 2026 11:07:19 -0700 Subject: [PATCH 15/55] fix new pandas read-only issue --- src/giant_python/bandsilo/baseline.py | 4 +++- tests/test_bandsilo_baseline.py | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/giant_python/bandsilo/baseline.py b/src/giant_python/bandsilo/baseline.py index 38e5ea8..3176f2a 100644 --- a/src/giant_python/bandsilo/baseline.py +++ b/src/giant_python/bandsilo/baseline.py @@ -144,7 +144,9 @@ def compute_f0( .to_numpy() ) - f0 = f0.reshape(f0.shape[0], -1) + # ``to_numpy()`` can hand back a read-only view (pandas copy-on-write), so + # take a writable, owned copy before assigning the per-column baselines. + f0 = np.array(f0, dtype=float).reshape(f0.shape[0], -1) for cix in range(f0.shape[1]): if np.all(np.isnan(f0[:, cix])): continue diff --git a/tests/test_bandsilo_baseline.py b/tests/test_bandsilo_baseline.py index 1caf62a..b6f2024 100644 --- a/tests/test_bandsilo_baseline.py +++ b/tests/test_bandsilo_baseline.py @@ -93,6 +93,24 @@ def test_sparse_hull_triggers_gap_interpolation(self): self.assertEqual(f0.shape, f.shape) self.assertFalse(np.all(np.isnan(f0))) + def test_copy_on_write_readonly_rolling_result(self): + """Under pandas copy-on-write, to_numpy() is read-only; f0 still fits. + + Regression: ``rolling().median().to_numpy()`` returns a read-only view + when copy-on-write is enabled (the default in newer pandas), which used + to break the in-place per-column baseline assignment. + """ + import pandas as pd + + f = np.stack( + [self._drifting_trace(seed=1), self._drifting_trace(seed=2)], + axis=1, + ) + with pd.option_context("mode.copy_on_write", True): + f0 = bl.compute_f0(f, 9, 40) + self.assertEqual(f0.shape, f.shape) + self.assertFalse(np.all(np.isnan(f0))) + class TestAssembleDff(unittest.TestCase): """assemble_dff combines the least-squares traces into dF/F.""" From 72e1918b61e077bdb8c9fdc9958b046ba35689a1 Mon Sep 17 00:00:00 2001 From: m-xie Date: Sat, 8 Aug 2026 05:54:56 +0000 Subject: [PATCH 16/55] use middle pixel as reference pixel in a band --- src/giant_python/bandsilo/geometry.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/giant_python/bandsilo/geometry.py b/src/giant_python/bandsilo/geometry.py index 89bc087..92e04e5 100644 --- a/src/giant_python/bandsilo/geometry.py +++ b/src/giant_python/bandsilo/geometry.py @@ -116,10 +116,7 @@ def build_subsample_matrix_inds( for sp in range(num_super_pixels): inds = np.where(sparse_mask_inds[:, 1] == sp + 1)[0] open_pixs = sparse_mask_inds[inds, 0] - 1 - if len(open_pixs) % 2 == 1: - ref_pix = int(np.median(open_pixs)) - else: - ref_pix = open_pixs[int(np.floor(len(open_pixs) / 2))] + ref_pix = open_pixs[int(np.floor(len(open_pixs) / 2))] out[sp, 0] = ref_pix out[sp, 1] = sp + 1 return out From e4aa69cb2173572cdd3662355a4390c6fef3ce87 Mon Sep 17 00:00:00 2001 From: Michael Xie Date: Fri, 7 Aug 2026 22:56:07 -0700 Subject: [PATCH 17/55] add missing default parameters --- src/giant_python/bandsilo/pipeline.py | 29 ++++++--------------------- src/giant_python/models/params.py | 25 ++++++++++++++++------- tests/test_bandsilo_pipeline.py | 14 ------------- tests/test_models.py | 9 +++++++++ 4 files changed, 33 insertions(+), 44 deletions(-) diff --git a/src/giant_python/bandsilo/pipeline.py b/src/giant_python/bandsilo/pipeline.py index 5ea14fe..71e31c2 100644 --- a/src/giant_python/bandsilo/pipeline.py +++ b/src/giant_python/bandsilo/pipeline.py @@ -437,27 +437,6 @@ def _resolve_params( return SiloParams(**params_in) -def effective_analyze_hz(analyze_hz) -> float: - """Return the band analysis rate (Hz), defaulting to 100 when unset. - - ``SiloParams.analyze_hz`` defaults to ``None`` (the reference's parameter - GUI always supplied a value, defaulting to 100 Hz). The band backend - materializes that fallback here so it is used consistently as the - high-res-trace sample rate and in the dF/F window sizes. - - Parameters - ---------- - analyze_hz : float or None - The configured analysis rate. - - Returns - ------- - float - ``analyze_hz`` as a float, or ``100.0`` when it is ``None``. - """ - return 100.0 if analyze_hz is None else float(analyze_hz) - - def _resolve_session_user_rois( result_dr: str, trial_table: dict, @@ -531,7 +510,6 @@ def extract_band_sources( The extracted sources and summary (also written to disk). """ params = _resolve_params(params_in) - params.analyze_hz = effective_analyze_hz(params.analyze_hz) log(f"Loading trial table {path_to_trial_table}", params.verbose) trial_table = load_trial_table(path_to_trial_table) result_dr = str(trial_table["savedr"]) @@ -556,6 +534,11 @@ def extract_band_sources( trial_table["align_hz"] = align_hz if params.num_channels is None: params.num_channels = num_channels + if params.num_channels is None: + raise ValueError( + "num_channels could not be determined from the alignment data " + "(no kept trials?); set SiloParams.num_channels explicitly." + ) user_rois = _resolve_session_user_rois( result_dr, trial_table, lookup, params @@ -961,7 +944,7 @@ def _localize( ) act_im = si.finalize_activity_image(act_im, sel_pix_idxs, nan_ct) - peak_th = params.peakth if params.peakth is not None else 8 + peak_th = params.peakth source_seeds = get_act_im_peaks( act_im, peak_th=peak_th, buffer_size=params.peak_buffer ) diff --git a/src/giant_python/models/params.py b/src/giant_python/models/params.py index 579066e..b52c270 100644 --- a/src/giant_python/models/params.py +++ b/src/giant_python/models/params.py @@ -48,6 +48,17 @@ class AlignParams: class SiloParams: """Source-extraction (SILo) parameters (mirrors the SILo params struct). + Defaults policy + --------------- + Every value parameter carries a concrete default, so a bare + :class:`SiloParams` never lets a ``None`` reach the numerics. ``None`` is + reserved as an explicit "resolve at runtime" sentinel for the few fields + that cannot have a fixed default: ``num_channels`` (read from the + acquisition metadata), ``interactive`` (auto-detected from the + environment), and the standard-SILo-only ``phi`` / ``tau_s`` / + ``photon_scale`` (consumed only by the not-yet-implemented pixel-movie + backend). Each of those is resolved in exactly one place. + Attributes ---------- microscope : str @@ -67,8 +78,8 @@ class SiloParams: Deconvolution time constant, in seconds. photon_scale : float or None Photons per digital unit (variance modeling). - peakth : float or None - Activity-image peak detection threshold. + peakth : float + Activity-image peak detection threshold (default ``8``). activity_channel : int Channel index used for activity detection. draw_user_rois : bool @@ -81,9 +92,9 @@ class SiloParams: guidance), and ``None`` (the default) auto-detects (respecting the ``GIANT_HEADLESS`` env var and stdin being a TTY). See :func:`giant_python.bandsilo.annotate.resolve_interactivity`. - analyze_hz : float or None - Temporal analysis rate, in Hz. When ``None``, the BandSILo backend - falls back to ``100``. + analyze_hz : float + Temporal analysis rate, in Hz (default ``100``; the value the + reference's parameter GUI defaulted to). Notes ----- @@ -130,11 +141,11 @@ class SiloParams: phi: Optional[float] = None tau_s: Optional[float] = None photon_scale: Optional[float] = None - peakth: Optional[float] = None + peakth: float = 8.0 activity_channel: int = 0 draw_user_rois: bool = False interactive: Optional[bool] = None - analyze_hz: Optional[float] = None + analyze_hz: float = 100.0 decay_tau_s: float = 0.15 baseline_window_s: float = 4.0 denoise_window_s: float = 1.0 diff --git a/tests/test_bandsilo_pipeline.py b/tests/test_bandsilo_pipeline.py index 8513c33..47326f3 100644 --- a/tests/test_bandsilo_pipeline.py +++ b/tests/test_bandsilo_pipeline.py @@ -77,20 +77,6 @@ def _path_result( ) -class TestEffectiveAnalyzeHz(unittest.TestCase): - """effective_analyze_hz applies the 100 Hz default for None.""" - - def test_none_defaults_to_100(self): - """An unset analyze_hz falls back to 100.0.""" - self.assertEqual(pl.effective_analyze_hz(None), 100.0) - - def test_value_passthrough(self): - """A set analyze_hz is returned as a float.""" - out = pl.effective_analyze_hz(80) - self.assertEqual(out, 80.0) - self.assertIsInstance(out, float) - - class TestDmdUserRois(unittest.TestCase): """_dmd_user_rois slices a resolved session selection per DMD.""" diff --git a/tests/test_models.py b/tests/test_models.py index fbda2ea..2830676 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -55,6 +55,15 @@ def test_param_models(self): self.assertEqual(SiloParams().microscope, "slap2") self.assertEqual(SiloParams().scan_mode, "standard") + def test_silo_value_params_have_concrete_defaults(self): + """Value params default to concrete (non-None) values.""" + p = SiloParams() + self.assertEqual(p.analyze_hz, 100.0) + self.assertEqual(p.peakth, 8.0) + # Only genuinely runtime-resolved fields stay None sentinels. + self.assertIsNone(p.num_channels) + self.assertIsNone(p.interactive) + class TestH5RoundTrip(unittest.TestCase): """from_h5/to_h5 are not implemented yet.""" From eaff16c246c6a1d38b547c31997fa7d23b4f7186 Mon Sep 17 00:00:00 2001 From: Michael Xie Date: Mon, 10 Aug 2026 01:50:34 -0700 Subject: [PATCH 18/55] restrict locations of final sources closer to peak --- src/giant_python/bandsilo/nmf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/giant_python/bandsilo/nmf.py b/src/giant_python/bandsilo/nmf.py index d4d05b5..d671a0f 100644 --- a/src/giant_python/bandsilo/nmf.py +++ b/src/giant_python/bandsilo/nmf.py @@ -468,8 +468,8 @@ def fit_gaussian_profiles( ) z_col = source_params[:, 0].unsqueeze(1) - loc_min = torch.from_numpy(source_seeds[:, 1:3] - d_xy).float() - loc_max = torch.from_numpy(source_seeds[:, 1:3] + d_xy).float() + loc_min = torch.from_numpy(source_seeds[:, 1:3] - 1.5).float() + loc_max = torch.from_numpy(source_seeds[:, 1:3] + 1.5).float() losses = [] shuffled_indices = None From 577def7efd2ee9f0a9dd8f39234f09755a0f54dd Mon Sep 17 00:00:00 2001 From: m-xie Date: Wed, 26 Aug 2026 23:00:26 +0000 Subject: [PATCH 19/55] add debugging feature to process only the first few trials --- src/giant_python/bandsilo/pipeline.py | 3 +++ src/giant_python/models/params.py | 6 ++++++ 2 files changed, 9 insertions(+) diff --git a/src/giant_python/bandsilo/pipeline.py b/src/giant_python/bandsilo/pipeline.py index 71e31c2..02e0134 100644 --- a/src/giant_python/bandsilo/pipeline.py +++ b/src/giant_python/bandsilo/pipeline.py @@ -602,6 +602,9 @@ def _process_dmd( soma_masks, soma_labels, soma_sps = _dmd_user_rois(user_rois, key) datadr = str(trial_table["datadr"]) n_trials = trial_table["filename"].shape[1] + if params.max_trials is not None: + n_trials = min(n_trials, params.max_trials) + log(f"Debug: limiting to first {n_trials} trial(s)", params.verbose) num_channels = params.num_channels align_hz = trial_table["align_hz"][key] keep_trials = trial_table["keep_trials"] diff --git a/src/giant_python/models/params.py b/src/giant_python/models/params.py index b52c270..c3792f6 100644 --- a/src/giant_python/models/params.py +++ b/src/giant_python/models/params.py @@ -132,6 +132,11 @@ class SiloParams: tqdm progress bars over its long-running loops (per-trial reads, rho, activity image, NMF). Off by default so headless / batch runs stay quiet. See :mod:`giant_python.bandsilo.progress`. + max_trials : int or None + Debug knob: when set, process only the first ``max_trials`` trials per + DMD (both the low-res read and high-res trace extraction). ``None`` + (the default) processes all trials. Use a small value (e.g. ``5``) to + iterate quickly while debugging. """ microscope: str = "slap2" @@ -158,6 +163,7 @@ class SiloParams: psf_dilation: int = 17 operator: str = "SLAP2 User" verbose: bool = False + max_trials: Optional[int] = None def set_params( From 013e227630dfc37e6a91b1fc9429df319a689350 Mon Sep 17 00:00:00 2001 From: m-xie Date: Wed, 26 Aug 2026 23:10:01 +0000 Subject: [PATCH 20/55] update dmd data reading bar progress bar --- src/giant_python/parallel.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/giant_python/parallel.py b/src/giant_python/parallel.py index 082fd12..4c0fe45 100644 --- a/src/giant_python/parallel.py +++ b/src/giant_python/parallel.py @@ -55,6 +55,15 @@ def map_trials( from joblib import Parallel, delayed - return Parallel(n_jobs=n_workers)( - delayed(func)(t) for t in _maybe_bar(trials, desc) + # ``return_as="generator"`` yields results (in submission order) as each + # trial *completes*, so wrapping this generator in tqdm tracks real + # progress. Wrapping the dispatch generator instead would fill the bar as + # soon as tasks are submitted, long before the workers finish. + results = Parallel(n_jobs=n_workers, return_as="generator")( + delayed(func)(t) for t in trials ) + if desc is not None: + from tqdm import tqdm + + results = tqdm(results, desc=desc, total=len(trials)) + return list(results) From f3fc3f788ce44bb8be7ff75d305ac9fbb2fc1663 Mon Sep 17 00:00:00 2001 From: m-xie Date: Fri, 28 Aug 2026 00:07:26 +0000 Subject: [PATCH 21/55] add bottleneck dependency and update rolling baseline to use median --- pyproject.toml | 1 + src/giant_python/bandsilo/background.py | 78 ++++++++++++++++--------- tests/test_bandsilo_background.py | 34 ++++++++--- 3 files changed, 80 insertions(+), 33 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1ade81f..64464e8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ dependencies = [ "torch", "opencv-python", "pandas", + "bottleneck", "slap2_utils @ git+https://github.com/m-xie/SLAP2_Utils.git@multidatafiles", ] diff --git a/src/giant_python/bandsilo/background.py b/src/giant_python/bandsilo/background.py index 4813c11..ed5af24 100644 --- a/src/giant_python/bandsilo/background.py +++ b/src/giant_python/bandsilo/background.py @@ -28,11 +28,17 @@ from typing import List, Tuple import numpy as np +import pandas as pd import scipy.ndimage as ndimage import torch from scipy import signal from scipy.interpolate import RectBivariateSpline +try: # pragma: no cover - exercised via the installed extra + import bottleneck as bn +except ImportError: # pragma: no cover + bn = None + from .progress import progress @@ -582,11 +588,19 @@ def baseline_window_frames(align_hz: float, baseline_window_s: float) -> int: def compute_rolling_baseline( interp_data: np.ndarray, baseline_window: int ) -> np.ndarray: - """Compute a NaN-aware rolling-mean baseline of the interpolated data. + """Compute a NaN-aware rolling-median baseline of the interpolated data. + + Each frame's baseline is the median of its centered window, ignoring NaNs, + so windows with no valid samples stay NaN and edge windows are truncated to + the available samples. This is the fast moving-median analogue of the + former moving-mean baseline; the median is more robust to transient + activity spikes when estimating the background. - Uses running window sums of values and of the valid-sample count, then - divides, so NaNs contribute nothing and windows with no valid samples stay - NaN. ``interp_data`` is modified in place (NaNs replaced by 0). + The heavy lifting uses :func:`bottleneck.move_median` (a C sliding-window + median whose cost is close to the moving mean) when available, falling back + to pandas' skip-list rolling median otherwise. ``bottleneck`` computes a + *trailing* window, so the data is right-padded with NaNs by ``window // 2`` + and the result is sliced back to center each window on its frame. Parameters ---------- @@ -598,31 +612,43 @@ def compute_rolling_baseline( Returns ------- ndarray - The rolling-mean background estimate (NaN where no valid samples). + The rolling-median background estimate (NaN where no valid samples). """ - valid = ~np.isnan(interp_data) - np.nan_to_num(interp_data, copy=False, nan=0.0) - - sum_vals = ( - ndimage.uniform_filter1d( - interp_data, size=baseline_window, axis=1, mode="nearest" - ) - * baseline_window - ) - valid_f = valid.astype(np.float32, copy=False) - count_vals = ( - ndimage.uniform_filter1d( - valid_f, size=baseline_window, axis=1, mode="nearest" + n_frames = interp_data.shape[1] + window = int(min(max(baseline_window, 1), n_frames)) + + if bn is not None: + pad = window // 2 + if pad: + padded = np.concatenate( + [ + interp_data, + np.full( + (interp_data.shape[0], pad), + np.nan, + dtype=interp_data.dtype, + ), + ], + axis=1, + ) + else: + padded = interp_data + trailing = bn.move_median( + padded, window=window, min_count=1, axis=1 ) - * baseline_window - ) - - interp_data_background = np.empty_like(sum_vals, dtype=interp_data.dtype) - interp_data_background.fill(np.nan) - np.divide( - sum_vals, count_vals, out=interp_data_background, where=count_vals > 0 + background = trailing[:, pad : pad + n_frames] + return background.astype(interp_data.dtype, copy=False) + + # Fallback (no bottleneck): pandas rolls along the row axis, so transpose + # to (n_frames, n_sel), roll each column, and transpose back. + background = ( + pd.DataFrame(interp_data.T) + .rolling(window=window, center=True, min_periods=1) + .median() + .to_numpy() + .T ) - return interp_data_background + return background.astype(interp_data.dtype, copy=False) def assemble_background( diff --git a/tests/test_bandsilo_background.py b/tests/test_bandsilo_background.py index f514097..10649d7 100644 --- a/tests/test_bandsilo_background.py +++ b/tests/test_bandsilo_background.py @@ -290,16 +290,36 @@ def test_window_frames(self): """Window length is int(hz * seconds).""" self.assertEqual(bg.baseline_window_frames(80.0, 4.0), 320) - def test_no_nan_matches_uniform_filter(self): - """With no NaNs, the baseline reduces to a plain moving average.""" - import scipy.ndimage as ndimage + def test_no_nan_matches_brute_median(self): + """With no NaNs, the baseline is a centered moving median.""" + + def brute_rolling_median(data, window): + n = data.shape[1] + out = np.full(data.shape, np.nan, dtype=float) + for r in range(data.shape[0]): + for i in range(n): + lo = max(0, i - (window - 1) // 2) + hi = min(n, i + window // 2 + 1) + vals = data[r, lo:hi] + vals = vals[~np.isnan(vals)] + if vals.size: + out[r, i] = np.median(vals) + return out data = np.arange(20, dtype=np.float32).reshape(2, 10) - expected = ndimage.uniform_filter1d( - data.copy(), size=3, axis=1, mode="nearest" - ) + for window in (3, 5): + expected = brute_rolling_median(data, window) + out = bg.compute_rolling_baseline(data.copy(), window) + np.testing.assert_allclose(out, expected, rtol=1e-5) + + def test_ignores_nan_in_window(self): + """NaNs are dropped from each window's median.""" + data = np.array([[1.0, np.nan, 3.0, 100.0, 5.0]], dtype=np.float32) out = bg.compute_rolling_baseline(data.copy(), 3) - np.testing.assert_allclose(out, expected, rtol=1e-5) + # window at idx 1 sees {1, 3} -> median 2; idx 2 sees {3, 100} -> 51.5 + np.testing.assert_allclose(out[0, 0], 1.0) + np.testing.assert_allclose(out[0, 1], 2.0) + np.testing.assert_allclose(out[0, 2], 51.5) def test_all_nan_row_stays_nan(self): """A fully-NaN row yields a fully-NaN baseline.""" From 8105ad30b185a0194e32a45b0018e6d1da7666da Mon Sep 17 00:00:00 2001 From: m-xie Date: Fri, 28 Aug 2026 08:32:50 +0000 Subject: [PATCH 22/55] fix sign issue when applying motion estimates --- src/giant_python/bandsilo/hdf5.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/src/giant_python/bandsilo/hdf5.py b/src/giant_python/bandsilo/hdf5.py index f95b9b4..41985a3 100644 --- a/src/giant_python/bandsilo/hdf5.py +++ b/src/giant_python/bandsilo/hdf5.py @@ -213,6 +213,14 @@ def load_alignment_data_h5(path: Union[str, Path]) -> dict: are stored as column vectors and online-motion shifts live under the nested ``slap2`` group in the new format; everything is flattened to 1-D here. + ``motionDSr``/``motionDSc``/``motionDSz`` are negated relative to the + on-disk values. The on-disk convention (shared by ``BandRegistration.m`` + and ``MultiRoiRegistration.m``) is the sample's physical displacement, + i.e. ``reference_position = canonical_position - motionDS_saved``. Every + bandsilo consumer instead writes ``reference_position = + canonical_position + motion``, so the sign is flipped once here to keep + that arithmetic correct everywhere downstream. + Parameters ---------- path : str or Path @@ -221,16 +229,28 @@ def load_alignment_data_h5(path: Union[str, Path]) -> dict: Returns ------- dict - Keys ``DSframes``, ``motionDSr/c/z``, ``onlineYshift/Xshift/Zshift``, - ``numChannels``, ``alignHz``. + Keys ``DSframes``, ``motionDSr/c/z`` (sign-flipped from the on-disk + values, see above), ``onlineYshift/Xshift/Zshift``, ``numChannels``, + ``alignHz``. """ s = load_struct_from_h5(path) slap2 = s.get("slap2", {}) or {} + # On disk, motionDSr/c/z is the sample displacement: BandRegistration.m + # searches a lookup table whose entry at offset `off` is + # ref(canonical + off), finds `motionDS_internal = off`, then saves + # `-motionDS_internal`. So `reference = canonical - motionDS_saved`. + # bandsilo consistently maps canonical superpixel coordinates into the + # reference stack as `canonical + motion` (see gui.py `sp_rows` and + # background.py `s_r`/`s_c`), which needs `motionDS_internal`. Negate + # back here, once, so that downstream arithmetic is correct. + motion_ds_r = _reshape_1d(s, "motionDSr") + motion_ds_c = _reshape_1d(s, "motionDSc") + motion_ds_z = _reshape_1d(s, "motionDSz") return { "DSframes": _reshape_1d(s, "DSframes"), - "motionDSr": _reshape_1d(s, "motionDSr"), - "motionDSc": _reshape_1d(s, "motionDSc"), - "motionDSz": _reshape_1d(s, "motionDSz"), + "motionDSr": None if motion_ds_r is None else -motion_ds_r, + "motionDSc": None if motion_ds_c is None else -motion_ds_c, + "motionDSz": None if motion_ds_z is None else -motion_ds_z, "onlineYshift": _reshape_1d(slap2, "onlineMotionYshift"), "onlineXshift": _reshape_1d(slap2, "onlineMotionXshift"), "onlineZshift": _reshape_1d(slap2, "onlineMotionZshift"), From 7e780f040f6e34f7e393337f0fa58895b000f21b Mon Sep 17 00:00:00 2001 From: m-xie Date: Mon, 31 Aug 2026 07:58:32 +0000 Subject: [PATCH 23/55] fix issue with double PSF convolution when doing matched filter --- src/giant_python/bandsilo/background.py | 89 +++++++++++++++++++++++++ src/giant_python/bandsilo/pipeline.py | 33 ++++++++- 2 files changed, 119 insertions(+), 3 deletions(-) diff --git a/src/giant_python/bandsilo/background.py b/src/giant_python/bandsilo/background.py index ed5af24..08227c1 100644 --- a/src/giant_python/bandsilo/background.py +++ b/src/giant_python/bandsilo/background.py @@ -285,6 +285,81 @@ def expand_psf( return psf_tensor, psf_center, psf_tensor_expanded, psf_center_expanded +def shrink_psf( + psf2d: np.ndarray, scale_y: float = 1.0, scale_x: float = 0.75 +) -> np.ndarray: + """Spatially shrink a PSF on the same grid (TEMPORARY). + + Resamples the PSF on a grid whose coordinates are divided by the per-axis + scale so a ``scale < 1`` narrows the PSF's spatial footprint along that + axis while keeping the same array shape and center; the result is + re-normalized to sum to 1. Used only for the temporary activity-image + experiment. + + Parameters + ---------- + psf2d : ndarray + The (cropped) PSF for this DMD. + scale_y : float + Spatial shrink factor along the row (y) axis (``< 1`` shrinks, + ``1`` is a no-op). + scale_x : float + Spatial shrink factor along the column (x) axis (``< 1`` shrinks, + ``1`` is a no-op). + + Returns + ------- + ndarray of shape ``psf2d.shape``, float32 + The shrunk, sum-normalized PSF. + """ + psf_arr = np.asarray(psf2d, dtype=np.float32) + psf_shape = psf_arr.shape + center_y = (psf_shape[0] - 1) / 2 + center_x = (psf_shape[1] - 1) / 2 + orig_y = np.arange(psf_shape[0], dtype=np.float64) - center_y + orig_x = np.arange(psf_shape[1], dtype=np.float64) - center_x + interp_spline = RectBivariateSpline(orig_y, orig_x, psf_arr) + shrunk = interp_spline(orig_y / scale_y, orig_x / scale_x) + shrunk = np.clip(shrunk, 0.0, None).astype(np.float32) + total = shrunk.sum() + if total > 0: + shrunk = shrunk / total + return shrunk + + +def gaussian_kernel_2d( + sigma: float, truncate: float = 4.0 +) -> Tuple[torch.Tensor, Tuple[int, int]]: + """Build a sum-normalized isotropic 2-D Gaussian kernel (TEMPORARY). + + Used by the temporary difference-of-Gaussians activity-image experiment, + which replaces the PSF-derived ``D``/``D_expanded`` convolution matrices + with an isotropic center/surround pair. The subtraction itself still + happens in :func:`compute_rho`, after per-column normalization. + + Parameters + ---------- + sigma : float + Gaussian standard deviation, in pixels. + truncate : float + Kernel half-width, in standard deviations. Truncation lowers the + realized standard deviation slightly, so the default is generous. + + Returns + ------- + kernel : torch.Tensor + Sum-normalized ``(2 * radius + 1, 2 * radius + 1)`` kernel. + center : tuple of int + ``(row, col)`` center index of ``kernel``. + """ + radius = int(np.ceil(truncate * sigma)) + coords = np.arange(-radius, radius + 1, dtype=np.float64) + g_1d = np.exp(-(coords**2) / (2.0 * sigma**2)) + kernel = np.outer(g_1d, g_1d).astype(np.float32) + kernel /= kernel.sum() + return torch.from_numpy(kernel), (radius, radius) + + def selected_pixels_2d_for_plane( sel_pix_idxs: np.ndarray, plane_z: int, @@ -1031,11 +1106,25 @@ def compute_rho( hd = torch.sparse.mm(h_sub, d_mats[z]) hd = hd / torch.sum(hd, dim=0, keepdim=True) + # Center-surround matched filter. Both projections are + # column-normalized before subtracting, so ``hd_diff`` has + # zero-sum columns and is blind to spatially flat residual. hd_expanded = torch.sparse.mm(h_sub, d_mats_expanded[z]) hd_expanded = hd_expanded / torch.sum( hd_expanded, dim=0, keepdim=True ) hd_diff = hd - hd_expanded + # TEMPORARY: whiten each filter column to unit L2 norm so every + # selected pixel's rho has the same noise variance. The L1 + # normalization above pins each column's *sum*, not its norm, so + # noise gain varies with where a column sits relative to the + # sparse superpixel reference-pixel lattice. Since the activity + # image accumulates rho^2 only at local maxima, that bias is + # amplified into a visible grid of bright reference pixels. + # Whitening also makes ``peak_th`` a true sigma threshold. + # To revert, delete the two lines below. + hd_norm = torch.linalg.norm(hd_diff, dim=0, keepdim=True) + hd_diff = hd_diff / hd_norm.clamp_min(1e-12) z_cols = z_col_idxs_np[z] z_valid_mask = valid_sel_cols[z_cols] diff --git a/src/giant_python/bandsilo/pipeline.py b/src/giant_python/bandsilo/pipeline.py index 02e0134..a13a9c4 100644 --- a/src/giant_python/bandsilo/pipeline.py +++ b/src/giant_python/bandsilo/pipeline.py @@ -897,7 +897,24 @@ def _localize( num_super_pixels, dmd_pixels_per_row, ) - psf_tensor, psf_center, psf_exp, psf_center_exp = bg.expand_psf(psf2d) + # TEMPORARY: build the activity-image D matrices from an isotropic + # difference-of-Gaussians instead of the PSF / expanded-PSF pair. + # ``compute_rho`` column-normalizes each and subtracts them, so the + # effective filter is H @ (G_center - G_surround). ``psf2d`` is still used + # there for the valid-column dilation/erosion mask. + # + # To revert, delete the Gaussian kernels below, restore these two lines + # (the ``shrink_psf`` call was itself a temporary hack and is optional), + # and swap the commented-out ``build_convolution_matrix`` calls back in + # inside the loop: + # psf2d = bg.shrink_psf(psf2d, scale_y=0.9, scale_x=0.75) + # psf_tensor, psf_center, psf_exp, psf_center_exp = bg.expand_psf(psf2d) + dog_center_sd = 0.9 + dog_surround_sd = 4.5 + center_kernel, center_kernel_ctr = bg.gaussian_kernel_2d(dog_center_sd) + surround_kernel, surround_kernel_ctr = bg.gaussian_kernel_2d( + dog_surround_sd + ) d_mats, d_mats_exp = [], [] for z in progress( range(num_fast_zs), @@ -907,11 +924,21 @@ def _localize( _, sel_2d = bg.selected_pixels_2d_for_plane( sel_pix_idxs, z, dmd_pixels_per_column, dmd_pixels_per_row ) + # d_mats.append( + # bg.build_convolution_matrix(sel_2d, psf_tensor, psf_center) + # ) + # d_mats_exp.append( + # bg.build_convolution_matrix(sel_2d, psf_exp, psf_center_exp) + # ) d_mats.append( - bg.build_convolution_matrix(sel_2d, psf_tensor, psf_center) + bg.build_convolution_matrix( + sel_2d, center_kernel, center_kernel_ctr + ) ) d_mats_exp.append( - bg.build_convolution_matrix(sel_2d, psf_exp, psf_center_exp) + bg.build_convolution_matrix( + sel_2d, surround_kernel, surround_kernel_ctr + ) ) rho = bg.compute_rho( residual, From aba7b8de2de7ebb8b30208ef224f5342676c6d48 Mon Sep 17 00:00:00 2001 From: m-xie Date: Mon, 31 Aug 2026 08:10:33 +0000 Subject: [PATCH 24/55] modify median subtraction --- src/giant_python/bandsilo/summary_images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/giant_python/bandsilo/summary_images.py b/src/giant_python/bandsilo/summary_images.py index 1ad60e7..7303357 100644 --- a/src/giant_python/bandsilo/summary_images.py +++ b/src/giant_python/bandsilo/summary_images.py @@ -273,7 +273,7 @@ def finalize_activity_image( ] = False act_im[nan_mask] = np.nan - med_act_im = ndimage.generic_filter(act_im, np.nanmedian, size=(1, 11, 11)) + med_act_im = ndimage.generic_filter(act_im, np.nanmedian, size=(1, 5, 11)) act_im = act_im - med_act_im act_im[nan_mask] = np.nan return act_im From 775eea624265e1d2a1e72731c40fdab9d55d9ae2 Mon Sep 17 00:00:00 2001 From: m-xie Date: Wed, 2 Sep 2026 23:39:24 +0000 Subject: [PATCH 25/55] use cubic interpolation as default for calculating background --- src/giant_python/bandsilo/background.py | 85 ++++++++++++++++++++++--- src/giant_python/bandsilo/pipeline.py | 2 + src/giant_python/models/params.py | 4 ++ tests/test_bandsilo_background.py | 46 +++++++++++++ tests/test_models.py | 1 + 5 files changed, 129 insertions(+), 9 deletions(-) diff --git a/src/giant_python/bandsilo/background.py b/src/giant_python/bandsilo/background.py index 08227c1..e8deb53 100644 --- a/src/giant_python/bandsilo/background.py +++ b/src/giant_python/bandsilo/background.py @@ -32,7 +32,7 @@ import scipy.ndimage as ndimage import torch from scipy import signal -from scipy.interpolate import RectBivariateSpline +from scipy.interpolate import CubicSpline, RectBivariateSpline try: # pragma: no cover - exercised via the installed extra import bottleneck as bn @@ -484,13 +484,24 @@ def _interp_columns( data, frames, dtype, + method="linear", + spline_weight_cache=None, ): - """Linearly interpolate each multi-point column onto its selected pixels. + """Interpolate each multi-point column onto its selected pixels. Selected rows that coincide with a reference row are copied exactly; those - strictly between two reference rows are linearly interpolated; those - outside the sampled row range are left untouched (NaN). + strictly between reference rows use linear or natural-cubic interpolation; + those outside the sampled row range are left untouched (NaN). Cubic spline + weights depend only on row geometry, so they are computed once and applied + to all complete frames in one matrix multiplication. Linear interpolation + is retained for columns with fewer than three samples and frames containing + missing source values. """ + if method not in {"linear", "cubic"}: + raise ValueError( + "interpolation method must be 'linear' or 'cubic', " + f"got {method!r}" + ) for c_int, rows in rows_by_col.items(): col_sel_pix_idxs = sel_idxs_by_col_all[c_int] target_rows = sel_rows[col_sel_pix_idxs] @@ -537,11 +548,53 @@ def _interp_columns( / (row_hi - row_lo) )[:, None] - vals_lo = data[src_idx[lo_v]][:, frames] - vals_hi = data[src_idx[hi_v]][:, frames] - out[np.ix_(col_sel_pix_idxs[interp_rows], frames)] = ( - 1.0 - alpha - ) * vals_lo + alpha * vals_hi + target_indices = col_sel_pix_idxs[interp_rows] + linear_frame_mask = np.ones(frames.size, dtype=bool) + has_cubic_geometry = rows.size >= 3 and np.all(np.diff(rows) > 0) + if method == "cubic" and has_cubic_geometry: + source_values = data[src_idx][:, frames] + complete_frames = np.all(np.isfinite(source_values), axis=0) + if np.any(complete_frames): + source_rows = rows.astype(dtype, copy=False) + target_interp_rows = target_rows[interp_rows].astype( + dtype, copy=False + ) + origin = source_rows[0] + cache_key = ( + tuple(source_rows - origin), + tuple(target_interp_rows - origin), + np.dtype(dtype).str, + ) + spline_weights = ( + spline_weight_cache.get(cache_key) + if spline_weight_cache is not None + else None + ) + if spline_weights is None: + spline_weights = CubicSpline( + source_rows, + np.eye(rows.size, dtype=dtype), + axis=0, + bc_type="natural", + extrapolate=False, + )(target_interp_rows).astype(dtype, copy=False) + if spline_weight_cache is not None: + spline_weight_cache[cache_key] = spline_weights + cubic_values = ( + spline_weights @ source_values[:, complete_frames] + ) + out[ + np.ix_(target_indices, frames[complete_frames]) + ] = cubic_values + linear_frame_mask = ~complete_frames + + if not np.any(linear_frame_mask): + continue + linear_frames = frames[linear_frame_mask] + vals_lo = data[src_idx[lo_v]][:, linear_frames] + vals_hi = data[src_idx[hi_v]][:, linear_frames] + linear_values = (1.0 - alpha) * vals_lo + alpha * vals_hi + out[np.ix_(target_indices, linear_frames)] = linear_values def build_interp_data( @@ -554,6 +607,7 @@ def build_interp_data( height: int = 800, width: int = 1280, dtype=np.float32, + method: str = "cubic", ) -> Tuple[np.ndarray, Tuple[int, int], Tuple[int, int]]: """Interpolate reference-pixel traces onto the selected-pixel grid. @@ -578,6 +632,11 @@ def build_interp_data( Sensor row/column extents used to window valid shifts. dtype : numpy dtype Working/output dtype. + method : {"linear", "cubic"} + Interpolation along each column (default ``"cubic"``). Cubic mode uses + a natural spline with precomputed geometry weights and falls back to + linear for columns with fewer than three samples or frames with missing + samples. Returns ------- @@ -591,6 +650,11 @@ def build_interp_data( ref_r = np.asarray(ref_r, dtype=np.int32) ref_c = np.asarray(ref_c, dtype=np.int32) data = np.asarray(data, dtype=dtype) + if method not in {"linear", "cubic"}: + raise ValueError( + "interpolation method must be 'linear' or 'cubic', " + f"got {method!r}" + ) r0 = int(max(0, ref_r.min()) + unique_motion_to_keep_yx[:, 0].min()) r1 = int(min(height, ref_r.max()) + unique_motion_to_keep_yx[:, 0].max()) @@ -610,6 +674,7 @@ def build_interp_data( np.flatnonzero(mot_inds_yx == idx) for idx in range(len(unique_motion_to_keep_yx)) ] + spline_weight_cache = {} if method == "cubic" else None for m_idx, frames in enumerate(frames_by_motion): if frames.size == 0: @@ -635,6 +700,8 @@ def build_interp_data( data, frames, dtype, + method, + spline_weight_cache, ) for rr_match, src_pos in single_points: out[np.ix_(rr_match, frames)] = data[src_pos, frames] diff --git a/src/giant_python/bandsilo/pipeline.py b/src/giant_python/bandsilo/pipeline.py index a13a9c4..8e3f35c 100644 --- a/src/giant_python/bandsilo/pipeline.py +++ b/src/giant_python/bandsilo/pipeline.py @@ -569,6 +569,7 @@ def _params_dict(params: SiloParams) -> dict: # pragma: no cover - glue "analyzeHz": params.analyze_hz, "denoiseWindow_s": params.denoise_window_s, "baselineWindow_s": params.baseline_window_s, + "backgroundInterpolation": params.background_interpolation, "decayTau_s": params.decay_tau_s, "dXY": params.d_xy, "sparse_fac": params.sparse_fac, @@ -845,6 +846,7 @@ def _estimate_background( sel_2d, umyx, mot_inds_yx, + method=params.background_interpolation, ) baseline_window = bg.baseline_window_frames( align_hz, params.baseline_window_s diff --git a/src/giant_python/models/params.py b/src/giant_python/models/params.py index c3792f6..0eb3861 100644 --- a/src/giant_python/models/params.py +++ b/src/giant_python/models/params.py @@ -107,6 +107,9 @@ class SiloParams: Calcium-decay time constant, in seconds. baseline_window_s : float Rolling-baseline window for dF/F, in seconds. + background_interpolation : str + Interpolation used to map sparse superpixels onto the background image: + ``"cubic"`` (default; natural cubic spline) or ``"linear"``. denoise_window_s : float Temporal denoising window, in seconds. vif : float @@ -153,6 +156,7 @@ class SiloParams: analyze_hz: float = 100.0 decay_tau_s: float = 0.15 baseline_window_s: float = 4.0 + background_interpolation: str = "cubic" denoise_window_s: float = 1.0 vif: float = 1.38 d_xy: int = 5 diff --git a/tests/test_bandsilo_background.py b/tests/test_bandsilo_background.py index 10649d7..41afb23 100644 --- a/tests/test_bandsilo_background.py +++ b/tests/test_bandsilo_background.py @@ -225,6 +225,52 @@ def test_basic_fill_and_bounds(self): self.assertAlmostEqual(float(out[2, 0]), 3.0, places=5) self.assertEqual(cb, (5, 5)) + def test_cubic_interpolation(self): + """Natural cubic interpolation is smooth and keeps exact samples.""" + ref_r = np.array([0, 2, 4, 6], dtype=np.int32) + ref_c = np.full(4, 5, dtype=np.int32) + data = np.array([[0.0], [1.0], [0.0], [1.0]], dtype=np.float32) + sel_2d = np.array([[0, 5], [1, 5], [2, 5], [3, 5]]) + umyx = np.array([[0, 0]], dtype=float) + mot_yx = np.array([0]) + linear, _, _ = bg.build_interp_data( + data, ref_r, ref_c, sel_2d, umyx, mot_yx, method="linear" + ) + cubic, _, _ = bg.build_interp_data( + data, ref_r, ref_c, sel_2d, umyx, mot_yx, method="cubic" + ) + self.assertAlmostEqual(float(cubic[0, 0]), 0.0, places=5) + self.assertAlmostEqual(float(cubic[2, 0]), 1.0, places=5) + self.assertNotAlmostEqual( + float(cubic[1, 0]), float(linear[1, 0]), places=5 + ) + + def test_cubic_falls_back_with_two_samples(self): + """Two-point columns retain the linear result in cubic mode.""" + ref_r = np.array([4, 6], dtype=np.int32) + ref_c = np.array([5, 5], dtype=np.int32) + data = np.array([[1.0], [3.0]], dtype=np.float32) + sel_2d = np.array([[5, 5]]) + umyx = np.array([[0, 0]], dtype=float) + mot_yx = np.array([0]) + out, _, _ = bg.build_interp_data( + data, ref_r, ref_c, sel_2d, umyx, mot_yx, method="cubic" + ) + self.assertAlmostEqual(float(out[0, 0]), 2.0, places=5) + + def test_unknown_interpolation_method_rejected(self): + """Interpolation method typos fail before processing data.""" + with self.assertRaisesRegex(ValueError, "linear.*cubic"): + bg.build_interp_data( + np.ones((1, 1)), + np.array([1]), + np.array([1]), + np.array([[1, 1]]), + np.array([[0, 0]]), + np.array([0]), + method="spline", + ) + def test_empty_motion_and_missing_column(self): """Empty bins and columns absent from the selection are skipped.""" ref_r = np.array([4, 6], dtype=np.int32) diff --git a/tests/test_models.py b/tests/test_models.py index 2830676..d0265a7 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -54,6 +54,7 @@ def test_param_models(self): self.assertEqual(AlignParams().align_hz, 80.0) self.assertEqual(SiloParams().microscope, "slap2") self.assertEqual(SiloParams().scan_mode, "standard") + self.assertEqual(SiloParams().background_interpolation, "cubic") def test_silo_value_params_have_concrete_defaults(self): """Value params default to concrete (non-None) values.""" From 89f29dc4a50ae283940bc68febc7e7a67583be94 Mon Sep 17 00:00:00 2001 From: m-xie Date: Thu, 3 Sep 2026 22:13:26 +0000 Subject: [PATCH 26/55] assume single pixel-sized sources for matched filter --- src/giant_python/bandsilo/pipeline.py | 41 +++++++++++++++++++-------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/src/giant_python/bandsilo/pipeline.py b/src/giant_python/bandsilo/pipeline.py index 8e3f35c..1b128b4 100644 --- a/src/giant_python/bandsilo/pipeline.py +++ b/src/giant_python/bandsilo/pipeline.py @@ -899,11 +899,12 @@ def _localize( num_super_pixels, dmd_pixels_per_row, ) - # TEMPORARY: build the activity-image D matrices from an isotropic - # difference-of-Gaussians instead of the PSF / expanded-PSF pair. + # TEMPORARY: build the activity-image D matrices from a one-pixel center + # and a 3x3 uniform surround instead of the PSF / expanded-PSF pair. # ``compute_rho`` column-normalizes each and subtracts them, so the - # effective filter is H @ (G_center - G_surround). ``psf2d`` is still used - # there for the valid-column dilation/erosion mask. + # effective filter is H @ (U_center - U_surround). ``psf2d`` is still used + # there for the valid-column dilation/erosion mask. The negative surround + # is normalized by its negative column sum before that subtraction. # # To revert, delete the Gaussian kernels below, restore these two lines # (the ``shrink_psf`` call was itself a temporary hack and is optional), @@ -911,12 +912,16 @@ def _localize( # inside the loop: # psf2d = bg.shrink_psf(psf2d, scale_y=0.9, scale_x=0.75) # psf_tensor, psf_center, psf_exp, psf_center_exp = bg.expand_psf(psf2d) - dog_center_sd = 0.9 - dog_surround_sd = 4.5 - center_kernel, center_kernel_ctr = bg.gaussian_kernel_2d(dog_center_sd) - surround_kernel, surround_kernel_ctr = bg.gaussian_kernel_2d( - dog_surround_sd - ) + # dog_center_sd = 0.9 + # dog_surround_sd = 4.5 + # center_kernel, center_kernel_ctr = bg.gaussian_kernel_2d(dog_center_sd) + # surround_kernel, surround_kernel_ctr = bg.gaussian_kernel_2d( + # dog_surround_sd + # ) + uniform_center_kernel = torch.ones((1, 1), dtype=torch.float32) + uniform_center_kernel_ctr = (0, 0) + uniform_surround_kernel = -torch.ones((3, 3), dtype=torch.float32) / 9.0 + uniform_surround_kernel_ctr = (1, 1) d_mats, d_mats_exp = [], [] for z in progress( range(num_fast_zs), @@ -932,14 +937,26 @@ def _localize( # d_mats_exp.append( # bg.build_convolution_matrix(sel_2d, psf_exp, psf_center_exp) # ) + # d_mats.append( + # bg.build_convolution_matrix( + # sel_2d, center_kernel, center_kernel_ctr + # ) + # ) + # d_mats_exp.append( + # bg.build_convolution_matrix( + # sel_2d, surround_kernel, surround_kernel_ctr + # ) + # ) d_mats.append( bg.build_convolution_matrix( - sel_2d, center_kernel, center_kernel_ctr + sel_2d, uniform_center_kernel, uniform_center_kernel_ctr ) ) d_mats_exp.append( bg.build_convolution_matrix( - sel_2d, surround_kernel, surround_kernel_ctr + sel_2d, + uniform_surround_kernel, + uniform_surround_kernel_ctr, ) ) rho = bg.compute_rho( From 0d20e9a4d5cb310c5fcc0bee87ec0d4a738a6e82 Mon Sep 17 00:00:00 2001 From: m-xie Date: Fri, 4 Sep 2026 22:31:30 +0000 Subject: [PATCH 27/55] revert to difference of gaussian filter --- src/giant_python/bandsilo/pipeline.py | 48 +++++++++++++-------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/src/giant_python/bandsilo/pipeline.py b/src/giant_python/bandsilo/pipeline.py index 1b128b4..0e02aed 100644 --- a/src/giant_python/bandsilo/pipeline.py +++ b/src/giant_python/bandsilo/pipeline.py @@ -912,16 +912,16 @@ def _localize( # inside the loop: # psf2d = bg.shrink_psf(psf2d, scale_y=0.9, scale_x=0.75) # psf_tensor, psf_center, psf_exp, psf_center_exp = bg.expand_psf(psf2d) - # dog_center_sd = 0.9 - # dog_surround_sd = 4.5 - # center_kernel, center_kernel_ctr = bg.gaussian_kernel_2d(dog_center_sd) - # surround_kernel, surround_kernel_ctr = bg.gaussian_kernel_2d( - # dog_surround_sd - # ) - uniform_center_kernel = torch.ones((1, 1), dtype=torch.float32) - uniform_center_kernel_ctr = (0, 0) - uniform_surround_kernel = -torch.ones((3, 3), dtype=torch.float32) / 9.0 - uniform_surround_kernel_ctr = (1, 1) + dog_center_sd = 0.9 + dog_surround_sd = 4.5 + center_kernel, center_kernel_ctr = bg.gaussian_kernel_2d(dog_center_sd) + surround_kernel, surround_kernel_ctr = bg.gaussian_kernel_2d( + dog_surround_sd + ) + # uniform_center_kernel = torch.ones((1, 1), dtype=torch.float32) + # uniform_center_kernel_ctr = (0, 0) + # uniform_surround_kernel = -torch.ones((3, 3), dtype=torch.float32) / 25.0 + # uniform_surround_kernel_ctr = (1, 1) d_mats, d_mats_exp = [], [] for z in progress( range(num_fast_zs), @@ -937,28 +937,28 @@ def _localize( # d_mats_exp.append( # bg.build_convolution_matrix(sel_2d, psf_exp, psf_center_exp) # ) - # d_mats.append( - # bg.build_convolution_matrix( - # sel_2d, center_kernel, center_kernel_ctr - # ) - # ) - # d_mats_exp.append( - # bg.build_convolution_matrix( - # sel_2d, surround_kernel, surround_kernel_ctr - # ) - # ) d_mats.append( bg.build_convolution_matrix( - sel_2d, uniform_center_kernel, uniform_center_kernel_ctr + sel_2d, center_kernel, center_kernel_ctr ) ) d_mats_exp.append( bg.build_convolution_matrix( - sel_2d, - uniform_surround_kernel, - uniform_surround_kernel_ctr, + sel_2d, surround_kernel, surround_kernel_ctr ) ) + # d_mats.append( + # bg.build_convolution_matrix( + # sel_2d, uniform_center_kernel, uniform_center_kernel_ctr + # ) + # ) + # d_mats_exp.append( + # bg.build_convolution_matrix( + # sel_2d, + # uniform_surround_kernel, + # uniform_surround_kernel_ctr, + # ) + # ) rho = bg.compute_rho( residual, mot_inds_yx, From fa8504a19e707280e302d29025a4f92e8b3fe60e Mon Sep 17 00:00:00 2001 From: m-xie Date: Wed, 9 Sep 2026 23:43:35 +0000 Subject: [PATCH 28/55] remove band artifact by median filtering along the superpixel pattern --- src/giant_python/bandsilo/pipeline.py | 27 +- src/giant_python/bandsilo/summary_images.py | 405 +++++++++++++++++++- src/giant_python/models/params.py | 5 + tests/test_bandsilo_phase_filter.py | 270 +++++++++++++ tests/test_bandsilo_pipeline.py | 67 ++++ tests/test_bandsilo_summary_images.py | 220 ++++++++++- 6 files changed, 968 insertions(+), 26 deletions(-) create mode 100644 tests/test_bandsilo_phase_filter.py diff --git a/src/giant_python/bandsilo/pipeline.py b/src/giant_python/bandsilo/pipeline.py index 0e02aed..fc46b86 100644 --- a/src/giant_python/bandsilo/pipeline.py +++ b/src/giant_python/bandsilo/pipeline.py @@ -899,14 +899,14 @@ def _localize( num_super_pixels, dmd_pixels_per_row, ) - # TEMPORARY: build the activity-image D matrices from a one-pixel center - # and a 3x3 uniform surround instead of the PSF / expanded-PSF pair. + # Build the activity-image D matrices from a narrow Gaussian center and a + # broad Gaussian surround instead of the PSF / expanded-PSF pair. # ``compute_rho`` column-normalizes each and subtracts them, so the - # effective filter is H @ (U_center - U_surround). ``psf2d`` is still used - # there for the valid-column dilation/erosion mask. The negative surround - # is normalized by its negative column sum before that subtraction. + # effective filter is H @ (G_center - G_surround). ``psf2d`` is still used + # there for the valid-column dilation/erosion mask. # - # To revert, delete the Gaussian kernels below, restore these two lines + # To restore the PSF-derived pair, delete the Gaussian kernels below, + # restore these two lines, # (the ``shrink_psf`` call was itself a temporary hack and is optional), # and swap the commented-out ``build_convolution_matrix`` calls back in # inside the loop: @@ -991,7 +991,20 @@ def _localize( dmd_pixels_per_row, verbose=params.verbose, ) - act_im = si.finalize_activity_image(act_im, sel_pix_idxs, nan_ct) + # Approximate the accumulated image's scan trajectory using the most + # populated retained YX motion bin. Dropped frames have bin index -1. + # Shift only the median geometry; rho/NMF still need the original centers. + motion_counts = np.bincount(mot_inds_yx[mot_inds_yx >= 0]) + dominant_motion = umyx[np.argmax(motion_counts)] + act_im = si.finalize_activity_image( + act_im, + sel_pix_idxs, + nan_ct, + ref_r=ref_r + int(dominant_motion[0]), + ref_c=ref_c + int(dominant_motion[1]), + normalize_mad=params.normalize_activity_mad, + ref_d=ref_d, + ) peak_th = params.peakth source_seeds = get_act_im_peaks( diff --git a/src/giant_python/bandsilo/summary_images.py b/src/giant_python/bandsilo/summary_images.py index 7303357..e146eff 100644 --- a/src/giant_python/bandsilo/summary_images.py +++ b/src/giant_python/bandsilo/summary_images.py @@ -13,8 +13,8 @@ The activity image is built by sweeping ``rho`` in temporal batches, finding spatio-temporal local maxima (a voxel greater than its 6 spatial neighbors in the same frame and its temporal neighbors), and accumulating their squared -value; it is then masked where data was never valid and has a local -(1x11x11) NaN-median subtracted. +value; it is then masked where data was never valid, has a local NaN-median +subtracted, and is divided by the raw local median absolute deviation (MAD). The reference's optional ``profile_activity_map`` timing instrumentation is intentionally dropped (it only prints timings and does not affect output). @@ -24,6 +24,7 @@ import numpy as np import scipy.ndimage as ndimage +from scipy.stats import median_abs_deviation from .progress import progress from .trial_data import fast_dilation @@ -244,27 +245,68 @@ def finalize_activity_image( act_im: np.ndarray, sel_pix_idxs: np.ndarray, nan_ct: np.ndarray, + ref_r: np.ndarray = None, + ref_c: np.ndarray = None, + normalize_mad: bool = True, + ref_d: np.ndarray = None, + phase_window_height: int = 7, + phase_window_width: int = 11, + phase_tolerance: float = 0.0, + phase_min_samples: int = 5, ) -> np.ndarray: - """Mask never-valid pixels and subtract a local NaN-median. + """Mask never-valid pixels and normalize by local NaN-median and MAD. Pixels whose rho was mostly NaN (``nan_ct > 0.5``) or that were never - selected are set to NaN; the remaining pixels have a local ``1x11x11`` - NaN-median subtracted, and the mask is re-applied. + selected are set to NaN. When superpixel reference centers are supplied, + an axis-aligned window includes only valid pixels with similar signed + vertical offsets from their nearest same-column reference center. + No snake identity or shared trajectory is assumed. Without reference + centers, a fixed 5x11 footprint is used. First-pass statistics use + the same local samples: ``MAD = nanmedian(abs(samples - median))``. + The output is ``(act_im - median) / MAD``, with no Gaussian scale factor. + Pixels with zero or non-finite MAD are set to NaN, as are masked pixels. + With ``normalize_mad=False``, only the median is subtracted; MAD is not + computed and zero-MAD pixels remain valid. With geometry, neighborhoods + below ``phase_min_samples`` use the mean of finite first-pass medians + (and MADs) in the full window, regardless of phase. Empty fallback + windows and pixels without reference geometry remain NaN. Parameters ---------- act_im : ndarray - Accumulated activity image (modified in place before subtraction). + Accumulated activity image (masked in place before normalization). sel_pix_idxs : ndarray of int Flattened selected-pixel indices into ``act_im``'s grid. nan_ct : ndarray of shape (n_sel,) Per-selected-pixel rho NaN fraction from :func:`giant_python.bandsilo.background.mask_high_nan_rho`. + ref_r, ref_c : ndarray, optional + Reference-center row and column for each superpixel, expressed in + the activity-image coordinate frame. The pipeline shifts these by + the dominant retained motion bin as an approximation for the + multi-motion image. Multiple bands can occupy the same column. + normalize_mad : bool + Divide the median-subtracted image by raw local MAD (default True). + False skips MAD computation and normalization, retaining masking and + median subtraction. + ref_d : ndarray, optional + Depth of each reference center. Required with geometry for multi-plane + images; omitted centers belong to plane zero in single-plane images. + phase_window_height, phase_window_width : int + Positive odd spatial window dimensions for phase matching (7x11). + phase_tolerance : float + Maximum signed-phase difference, in row pixels (default 0, exact + equality). Midpoint ties choose the smaller reference row. + phase_min_samples : int + Minimum finite phase-matched samples for first-pass median/MAD + (default 5); smaller neighborhoods use the full-window fallback. Returns ------- ndarray - The finalized activity image (NaN outside the valid support). + Activity in raw local-MAD units (NaN outside the valid support or + where the local MAD is zero or non-finite), or median-subtracted + activity in original units when ``normalize_mad=False``. """ nan_mask = np.full_like(act_im, True, dtype=bool) valid_sel_pix = np.flatnonzero(nan_ct <= 0.5) @@ -273,7 +315,352 @@ def finalize_activity_image( ] = False act_im[nan_mask] = np.nan - med_act_im = ndimage.generic_filter(act_im, np.nanmedian, size=(1, 5, 11)) + if ref_r is not None and ref_c is not None: + # Legacy trajectory-following filter: uncomment this call and comment + # out the phase-matched call below to switch back. The original helper + # is retained unchanged below for comparison and regression tests. + # local_stats = snake_aligned_nanmedian( + # act_im, np.flatnonzero(~nan_mask), ref_r, ref_c, + # return_mad=normalize_mad, + # ) + local_stats = phase_matched_nanmedian( + act_im, + np.flatnonzero(~nan_mask), + ref_r, + ref_c, + ref_d=ref_d, + height=phase_window_height, + width=phase_window_width, + phase_tolerance=phase_tolerance, + min_samples=phase_min_samples, + return_mad=normalize_mad, + ) + if normalize_mad: + med_act_im, mad_act_im = local_stats + else: + med_act_im = local_stats + else: + med_act_im = ndimage.generic_filter( + act_im, np.nanmedian, size=(1, 5, 11) + ) + if normalize_mad: + # Use each footprint's own median, not the spatial median image. + mad_act_im = ndimage.generic_filter( + act_im, + median_abs_deviation, + size=(1, 5, 11), + extra_keywords={"nan_policy": "omit"}, + ) act_im = act_im - med_act_im - act_im[nan_mask] = np.nan + if not normalize_mad: + act_im[nan_mask] = np.nan + return act_im + valid_scale = (~nan_mask) & np.isfinite(mad_act_im) & (mad_act_im > 0) + np.divide(act_im, mad_act_im, out=act_im, where=valid_scale) + act_im[~valid_scale] = np.nan return act_im + + +def phase_matched_nanmedian( + image: np.ndarray, + valid_pix_idxs: np.ndarray, + ref_r: np.ndarray, + ref_c: np.ndarray, + ref_d: np.ndarray = None, + height: int = 7, + width: int = 11, + phase_tolerance: float = 0.0, + min_samples: int = 5, + chunk_size: int = 4096, + return_mad: bool = False, +) -> np.ndarray | tuple[np.ndarray, np.ndarray]: + """Local median/MAD conditioned on signed reference-band phase. + + For each selected pixel, phase is ``row - nearest_reference_row`` using + only reference centers in the same column and depth. Distances are + vertical, not Euclidean, so adjacent-column center spacing cannot change + the sign/phase. Different bands may overlap in column space and have + different curvature. Reference coordinates must already include motion. + + All nearest-center assignments are retained; exact distance ties choose + the smaller reference row. Duplicate centers are deduplicated; exact + coincident branches cannot be distinguished from these coordinates alone. + Columns without centers have no phase: no interpolation across gaps or + extrapolation beyond scanned columns is attempted. Multi-plane images + require ``ref_d`` to avoid borrowing geometry from a different plane. + + Within a clipped, axis-aligned ``height x width`` window, take only finite + pixels in ``valid_pix_idxs`` whose signed phase differs by at most + ``phase_tolerance`` (default 0, exact equality). The center itself is + included if finite. There is no + branch-membership restriction: nearby different snakes can contribute + when their phases match. At least ``min_samples`` (default 5) are required + for first-pass statistics. Otherwise, use the nanmean of finite first-pass + medians in the full spatial window, regardless of phase. Fallback values + are never reused by other windows; an empty fallback remains NaN. + This is a phase-conditioned spatial background, not a + reconstruction or separation of contributions at physical crossings. + + With ``return_mad=True``, first-pass raw MAD uses exactly the median's + samples, without Gaussian scaling. Fallback MAD is the nanmean of finite + first-pass MADs in the full window, not a pooled-sample MAD. Return both + images; otherwise skip MAD entirely. Non-selected/nonfinite pixels and + pixels without geometry remain NaN. Sparse phase/support arrays and chunked + gathers bound working memory; only output images are dense. + """ + image = np.asarray(image) + if image.ndim != 3 or any(size == 0 for size in image.shape): + raise ValueError("image must have non-empty shape (depth, row, column)") + for name, value in (("height", height), ("width", width)): + if not isinstance(value, (int, np.integer)) or value < 1 or value % 2 == 0: + raise ValueError(f"{name} must be a positive odd integer") + for name, value in (("min_samples", min_samples), ("chunk_size", chunk_size)): + if not isinstance(value, (int, np.integer)) or value < 1: + raise ValueError(f"{name} must be a positive integer") + if not np.isfinite(phase_tolerance) or phase_tolerance < 0: + raise ValueError("phase_tolerance must be finite and nonnegative") + + ref_r = np.asarray(ref_r, dtype=float).reshape(-1) + ref_c = np.asarray(ref_c).reshape(-1) + if not ref_r.size or ref_r.size != ref_c.size: + raise ValueError("ref_r and ref_c must be non-empty and equally sized") + if not np.all(np.isfinite(ref_r)): + raise ValueError("reference rows must be finite") + if ref_d is None: + if image.shape[0] != 1: + raise ValueError("ref_d is required for multi-plane images") + ref_d = np.zeros(ref_r.size, dtype=np.intp) + else: + ref_d = np.asarray(ref_d).reshape(-1) + if ref_d.size != ref_r.size: + raise ValueError("ref_d must have the same size as ref_r") + for coordinates in (ref_c, ref_d): + if not np.all(np.isfinite(coordinates)) or np.any(coordinates != np.rint(coordinates)): + raise ValueError("reference columns and depths must be finite integers") + ref_c, ref_d = ref_c.astype(np.intp), ref_d.astype(np.intp) + + selected = np.asarray(valid_pix_idxs) + if selected.ndim != 1 or (selected.size and not np.issubdtype(selected.dtype, np.integer)): + raise ValueError("valid_pix_idxs must be a one-dimensional integer array") + if np.any(selected < 0) or np.any(selected >= image.size): + raise ValueError("valid_pix_idxs contains an out-of-bounds index") + selected = np.unique(selected.astype(np.intp)) + result = np.full(image.shape, np.nan, dtype=np.result_type(image.dtype, np.float32)) + mad_result = np.full_like(result, np.nan) if return_mad else None + if not selected.size: + return (result, mad_result) if return_mad else result + + nz, nr, nc = image.shape + in_bounds = (ref_d >= 0) & (ref_d < nz) & (ref_c >= 0) & (ref_c < nc) + # Group references by (depth, column); unlike the legacy filter, do not + # collapse all rows in a column to one shared trajectory. + references = {} + for depth, column, row in zip(ref_d[in_bounds], ref_c[in_bounds], ref_r[in_bounds]): + references.setdefault(int(depth * nc + column), []).append(row) + depths, rows, cols = np.unravel_index(selected, image.shape) + keys = depths * nc + cols + order = np.argsort(keys, kind="stable") + boundaries = np.r_[0, np.flatnonzero(np.diff(keys[order])) + 1, selected.size] + phase = np.full(selected.size, np.nan) + for start, stop in zip(boundaries[:-1], boundaries[1:]): + positions = order[start:stop] + centers = references.get(int(keys[positions[0]])) + if centers is None: + continue + centers = np.unique(centers) + pixel_rows = rows[positions] + insertion = np.searchsorted(centers, pixel_rows) + left = np.clip(insertion - 1, 0, centers.size - 1) + right = np.clip(insertion, 0, centers.size - 1) + nearest = np.where( + np.abs(pixel_rows - centers[left]) <= np.abs(pixel_rows - centers[right]), + left, right, + ) + phase[positions] = pixel_rows - centers[nearest] + + # Lookup both support membership and phase using sorted selected indices; + # finite unselected image pixels must never enter the sample population. + row_offsets, col_offsets = np.meshgrid( + np.arange(-(height // 2), height // 2 + 1), + np.arange(-(width // 2), width // 2 + 1), indexing="ij", + ) + row_offsets, col_offsets = row_offsets.ravel(), col_offsets.ravel() + values = image.ravel()[selected] + outputs = np.flatnonzero(np.isfinite(phase) & np.isfinite(values)) + + def windows(output_positions): + for start in range(0, output_positions.size, chunk_size): + positions = output_positions[start:start + chunk_size] + sample_r = rows[positions, None] + row_offsets + sample_c = cols[positions, None] + col_offsets + inside = (sample_r >= 0) & (sample_r < nr) & (sample_c >= 0) & (sample_c < nc) + sample_flat = (depths[positions, None] * nr + sample_r) * nc + sample_c + lookup = np.minimum(np.searchsorted(selected, sample_flat), selected.size - 1) + yield positions, lookup, inside & (selected[lookup] == sample_flat) + + for positions, lookup, support in windows(outputs): + accept = ( + support + & np.isfinite(values[lookup]) + & (np.abs(phase[lookup] - phase[positions, None]) <= phase_tolerance) + ) + enough = np.count_nonzero(accept, axis=1) >= min_samples + if not np.any(enough): + continue + samples = np.where(accept[enough], values[lookup[enough]], np.nan).astype(result.dtype) + medians = np.nanmedian(samples, axis=1) + flat = selected[positions[enough]] + result.ravel()[flat] = medians + if return_mad: + np.subtract(samples, medians[:, None], out=samples) + np.abs(samples, out=samples) + mad_result.ravel()[flat] = np.nanmedian(samples, axis=1) + + # Advanced indexing freezes first-pass values: fallback estimates must + # not propagate or depend on chunk size / iteration order. + first_medians = result.ravel()[selected] + first_mads = mad_result.ravel()[selected] if return_mad else None + missing = outputs[~np.isfinite(first_medians[outputs])] + statistics = [(first_medians, result)] + if return_mad: + statistics.append((first_mads, mad_result)) + for positions, lookup, support in windows(missing): + for source, destination in statistics: + accept = support & np.isfinite(source[lookup]) + counts = np.count_nonzero(accept, axis=1) + enough = counts > 0 + # Equivalent to nanmean, without empty-window warnings. + totals = np.sum(np.where(accept, source[lookup], 0), axis=1, dtype=np.float64) + destination.ravel()[selected[positions[enough]]] = totals[enough] / counts[enough] + return (result, mad_result) if return_mad else result + + +def snake_aligned_nanmedian( + image: np.ndarray, + valid_pix_idxs: np.ndarray, + ref_r: np.ndarray, + ref_c: np.ndarray, + height: int = 5, + width: int = 11, + chunk_size: int = 100_000, + return_mad: bool = False, +) -> np.ndarray | tuple[np.ndarray, np.ndarray]: + """Return local medians and optionally MADs along the band trajectory. + + The parallel snakes have the same trajectory and differ by a nearly fixed + vertical spacing. The trajectory is unwrapped modulo that spacing so its + row does not jump when an outer snake is absent from a column. Disconnected + column runs are kept separate. For each valid output pixel, this function + gathers ``height * width`` samples translated by the local trajectory. + Work is restricted to valid pixels and chunked to bound temporary memory. + ``ref_r`` and ``ref_c`` must already be expressed in ``image`` coordinates; + this helper does not apply motion correction itself. + + With ``return_mad=True``, return ``(median_image, mad_image)``. The raw + MAD is ``nanmedian(abs(samples - median))`` over exactly the same samples + used for each output median, without Gaussian scaling. The sample buffer + is reused for absolute deviations. Both images are NaN at unrequested + pixels and all-NaN footprints; constant footprints have MAD zero. + The default returns only the median image for compatibility. + """ + if height < 1 or width < 1 or height % 2 == 0 or width % 2 == 0: + raise ValueError("height and width must be positive odd integers") + + ref_r = np.asarray(ref_r).reshape(-1) + ref_c = np.asarray(ref_c).reshape(-1).astype(np.intp, copy=False) + if ref_r.size == 0 or ref_r.size != ref_c.size: + raise ValueError("ref_r and ref_c must be non-empty and equally sized") + + n_rows, n_cols = image.shape[-2:] + in_bounds = (ref_c >= 0) & (ref_c < n_cols) + columns = np.unique(ref_c[in_bounds]) + if columns.size == 0: + raise ValueError("no reference centers fall inside the image") + + rows_by_column = [np.sort(ref_r[ref_c == column]) for column in columns] + spacing_samples = [ + np.diff(rows) for rows in rows_by_column if rows.size > 1 + ] + row_spacings = ( + np.concatenate(spacing_samples) + if spacing_samples + else np.empty(0, dtype=float) + ) + snake_spacing = ( + float(np.median(row_spacings[row_spacings > 0])) + if np.any(row_spacings > 0) + else None + ) + + run_starts = np.r_[0, np.flatnonzero(np.diff(columns) > 1) + 1] + run_stops = np.r_[run_starts[1:], columns.size] + center_rows = np.array([rows[0] for rows in rows_by_column], dtype=float) + segment_at_reference = np.empty(columns.size, dtype=np.intp) + for segment, (start, stop) in enumerate(zip(run_starts, run_stops)): + segment_at_reference[start:stop] = segment + if snake_spacing is None: + center_rows[start:stop] = [ + np.median(rows) for rows in rows_by_column[start:stop] + ] + continue + for index in range(start + 1, stop): + delta = center_rows[index] - center_rows[index - 1] + center_rows[index] -= np.rint(delta / snake_spacing) * snake_spacing + + all_columns = np.arange(n_cols) + insertions = np.searchsorted(columns, all_columns) + right = np.clip(insertions, 0, columns.size - 1) + left = np.clip(insertions - 1, 0, columns.size - 1) + nearest = np.where( + np.abs(all_columns - columns[left]) + <= np.abs(columns[right] - all_columns), + left, + right, + ) + trajectory = center_rows[nearest] + trajectory_segment = segment_at_reference[nearest] + + result = np.full_like(image, np.nan, dtype=np.result_type(image, np.float32)) + mad_result = np.full_like(result, np.nan) if return_mad else None + col_offsets = np.arange(-(width // 2), width // 2 + 1) + row_offsets = np.arange(-(height // 2), height // 2 + 1) + + for start in range(0, len(valid_pix_idxs), chunk_size): + flat = valid_pix_idxs[start : start + chunk_size] + depths, rows, cols = np.unravel_index(flat, image.shape) + sample_cols = cols[:, None] + col_offsets[None, :] + valid_cols = (sample_cols >= 0) & (sample_cols < n_cols) + clipped_cols = np.clip(sample_cols, 0, n_cols - 1) + valid_cols &= ( + trajectory_segment[clipped_cols] + == trajectory_segment[cols, None] + ) + path_delta = np.rint( + trajectory[clipped_cols] - trajectory[cols, None] + ).astype(np.intp) + path_rows = rows[:, None] + path_delta + + sample_rows = path_rows[:, :, None] + row_offsets[None, None, :] + sample_cols = np.broadcast_to( + clipped_cols[:, :, None], sample_rows.shape + ) + sample_depths = np.broadcast_to( + depths[:, None, None], sample_rows.shape + ) + valid = ( + valid_cols[:, :, None] + & (sample_rows >= 0) + & (sample_rows < n_rows) + ) + samples = np.full(sample_rows.shape, np.nan, dtype=result.dtype) + samples[valid] = image[ + sample_depths[valid], sample_rows[valid], sample_cols[valid] + ] + medians = np.nanmedian(samples, axis=(1, 2)) + result.ravel()[flat] = medians + if return_mad: + np.subtract(samples, medians[:, None, None], out=samples) + np.abs(samples, out=samples) + mad_result.ravel()[flat] = np.nanmedian(samples, axis=(1, 2)) + + return (result, mad_result) if return_mad else result diff --git a/src/giant_python/models/params.py b/src/giant_python/models/params.py index 0eb3861..6ac2858 100644 --- a/src/giant_python/models/params.py +++ b/src/giant_python/models/params.py @@ -140,6 +140,10 @@ class SiloParams: DMD (both the low-res read and high-res trace extraction). ``None`` (the default) processes all trials. Use a small value (e.g. ``5``) to iterate quickly while debugging. + normalize_activity_mad : bool + Band-scan activity images are normalized by raw local MAD by default. + False retains local median subtraction but skips MAD computation and + division. """ microscope: str = "slap2" @@ -168,6 +172,7 @@ class SiloParams: operator: str = "SLAP2 User" verbose: bool = False max_trials: Optional[int] = None + normalize_activity_mad: bool = True def set_params( diff --git a/tests/test_bandsilo_phase_filter.py b/tests/test_bandsilo_phase_filter.py new file mode 100644 index 0000000..b093121 --- /dev/null +++ b/tests/test_bandsilo_phase_filter.py @@ -0,0 +1,270 @@ +"""Phase-matched background statistics, including overlapping band columns.""" + +import unittest + +import numpy as np + +from giant_python.bandsilo import summary_images as si + + +def brute_stats(image, selected, ref_r, ref_c, ref_d, height, width, + tolerance, minimum): + """Independent small-grid reference using explicit distance/window loops.""" + phases = np.full(image.shape, np.nan) + med = np.full(image.shape, np.nan) + mad = np.full(image.shape, np.nan) + for flat in selected: + z, r, c = np.unravel_index(flat, image.shape) + centers = np.unique(ref_r[(ref_d == z) & (ref_c == c)]) + if not centers.size: + continue + distances = np.abs(r - centers) + phases[z, r, c] = r - centers[np.argmin(distances)] + for flat in selected: + z, r, c = np.unravel_index(flat, image.shape) + if not np.isfinite(phases[z, r, c]) or not np.isfinite(image[z, r, c]): + continue + samples = [] + for sr in range(max(0, r - height // 2), min(image.shape[1], r + height // 2 + 1)): + for sc in range(max(0, c - width // 2), min(image.shape[2], c + width // 2 + 1)): + if abs(phases[z, sr, sc] - phases[z, r, c]) <= tolerance and np.isfinite(image[z, sr, sc]): + samples.append(image[z, sr, sc]) + if len(samples) >= minimum: + med[z, r, c] = np.median(samples) + mad[z, r, c] = np.median(np.abs(samples - med[z, r, c])) + first_med, first_mad = med.copy(), mad.copy() + for flat in selected: + z, r, c = np.unravel_index(flat, image.shape) + if (not np.isfinite(phases[z, r, c]) or not np.isfinite(image[z, r, c]) + or np.isfinite(first_med[z, r, c])): + continue + for source, target in ((first_med, med), (first_mad, mad)): + window = source[z, max(0, r - height // 2):r + height // 2 + 1, + max(0, c - width // 2):c + width // 2 + 1] + finite = window[np.isfinite(window)] + if finite.size: + target[z, r, c] = np.mean(finite) + return med, mad + + +class TestPhaseMatchedFilter(unittest.TestCase): + def test_overlapping_curved_bands_match_brute_force(self): + rng = np.random.default_rng(8) + image = rng.normal(size=(2, 23, 17)).astype(np.float32) + # Independent, nonparallel trajectories with overlapping column ranges. + cols = np.arange(17) + ref_c = np.tile(cols, 4) + ref_r = np.r_[4 + abs(cols - 8) // 2, 16 - abs(cols - 8) // 2, + 3 + cols // 3, 19 - cols // 3].astype(float) + ref_d = np.repeat([0, 1], 34) + keep = ref_c != 7 # Column gap is not interpolated. + ref_r, ref_c, ref_d = ref_r[keep], ref_c[keep], ref_d[keep] + selected = np.flatnonzero(rng.random(image.size) > 0.15) + image.ravel()[selected[::29]] = np.nan + image.ravel()[selected[::41]] = np.inf + original = image.copy() + expected = brute_stats(image, selected, ref_r, ref_c, ref_d, 9, 7, 0, 3) + for chunk_size in (1, 53, 4096): + actual = si.phase_matched_nanmedian( + image, selected[::-1], ref_r, ref_c, ref_d, + height=9, width=7, min_samples=3, + chunk_size=chunk_size, return_mad=True, + ) + for values, truth in zip(actual, expected): + np.testing.assert_allclose(values, truth, atol=1e-6, equal_nan=True) + self.assertTrue(np.isnan(values[:, :, 7]).all()) + np.testing.assert_array_equal(image, original) + + def test_signed_phase_not_absolute_distance(self): + image = np.zeros((1, 13, 11), dtype=np.float32) + cols = np.arange(11) + centers = np.full(11, 6) + image[0, 5, :] = 10 # phase -1 + image[0, 7, :] = 100 # phase +1 + selected = np.arange(image.size) + med, mad = si.phase_matched_nanmedian( + image, selected, centers, cols, return_mad=True + ) + self.assertEqual(med[0, 5, 5], 10) + self.assertEqual(med[0, 7, 5], 100) + self.assertEqual(mad[0, 5, 5], 0) + + def test_phase_tolerance_and_minimum_samples(self): + image = np.arange(5 * 7, dtype=float).reshape(1, 5, 7) + cols = np.arange(7) + centers = 2 + 0.25 * (cols % 2) + selected = np.arange(image.size) + for tolerance in (0, 0.3, 1.0): + for minimum in (1, 5, 100): + truth = brute_stats(image, selected, centers, cols, np.zeros(7), + 5, 7, tolerance, minimum) + result = si.phase_matched_nanmedian( + image, selected, centers, cols, height=5, width=7, + phase_tolerance=tolerance, min_samples=minimum, + return_mad=True, + ) + for actual, expected in zip(result, truth): + np.testing.assert_allclose(actual, expected, equal_nan=True) + + def test_midpoint_rows_and_duplicate_centers_are_retained(self): + image = np.ones((1, 16, 11)) + cols = np.tile(np.arange(11), 3) + centers = np.repeat([4, 4, 12], 11) # Deduplicate the top band. + selected = np.arange(image.size) + med = si.phase_matched_nanmedian(image, selected, centers, cols) + np.testing.assert_array_equal(med[0, 8], 1) # Equidistant bands. + self.assertTrue(np.isfinite(med[0, 4]).all()) + centers = np.repeat([4, 4, 13], 11) + med = si.phase_matched_nanmedian(image, selected, centers, cols) + np.testing.assert_array_equal(med[0, 8:10], 1) # Near ties also retained. + + def test_midpoint_tie_chooses_smaller_reference_row(self): + image = np.array([[[10., 20., 30., 40., 50., 100., 200.]]]) + cols = np.arange(7) + # First five pixels are ties with signed phase +4; last two are -4. + lower = np.r_[np.full(5, -4), np.full(2, -14)] + upper = np.full(7, 4) + selected = np.arange(image.size) + med = si.phase_matched_nanmedian( + image, selected, np.r_[lower, upper], np.tile(cols, 2), + ) + self.assertEqual(med[0, 0, 3], 30) + + def test_default_exact_phase_does_not_pool_nearby_offsets(self): + image = np.full((1, 4, 11), np.nan) + cols = np.arange(11) + centers = np.where(cols % 2, 1.25, 1.0) + image[0, 2, :] = np.where(cols % 2, 100, 10) + selected = np.flatnonzero(np.isfinite(image)) + med = si.phase_matched_nanmedian(image, selected, centers, cols) + self.assertEqual(med[0, 2, 5], 100) # Five exact +0.75 matches. + self.assertEqual(med[0, 2, 4], 10) # Five exact +1 matches. + tolerant = si.phase_matched_nanmedian( + image, selected, centers, cols, phase_tolerance=0.5, + ) + self.assertEqual(tolerant[0, 2, 5], 10) + + def test_irregular_spacing_keeps_pixel_offsets_not_fractional_phase(self): + image = np.full((1, 9, 7), np.nan) + cols = np.arange(7) + gaps = np.array([8, 12, 8, 12, 8, 12, 8]) + image[0, 3, :] = np.arange(1, 8) # +1 in every column. + refs = np.r_[np.full(7, 2), 2 + gaps] + med = si.phase_matched_nanmedian( + image, np.flatnonzero(np.isfinite(image)), refs, np.tile(cols, 2), + ) + self.assertEqual(med[0, 3, 3], 4) + + def test_fallback_uses_first_pass_statistics_not_raw_or_filled_values(self): + image = np.full((1, 3, 11), np.nan, dtype=np.float32) + image[0, 0] = np.arange(11) ** 2 + image[0, 1:3, 5] = 9999 + selected = np.flatnonzero(np.isfinite(image)) + cols = np.arange(11) + expected_med, expected_mad = [], [] + for col in cols: + samples = image[0, 0, max(0, col - 5):col + 6] + median = np.median(samples) + expected_med.append(median) + expected_mad.append(np.median(abs(samples - median))) + for chunk in (1, 4, 4096): + med, mad = si.phase_matched_nanmedian( + image, selected[::-1], np.zeros(11), cols, + height=3, chunk_size=chunk, return_mad=True, + ) + np.testing.assert_allclose(med[0, 0], expected_med) + np.testing.assert_allclose(mad[0, 0], expected_mad) + self.assertAlmostEqual(float(med[0, 1, 5]), np.mean(expected_med), places=5) + self.assertAlmostEqual(float(mad[0, 1, 5]), np.mean(expected_mad), places=5) + # Filled row 1 cannot propagate into row 2, even in later chunks. + self.assertTrue(np.isnan(med[0, 2, 5])) + self.assertTrue(np.isnan(mad[0, 2, 5])) + self.assertTrue(np.isnan(med[~np.isfinite(image)]).all()) + median_only = si.phase_matched_nanmedian( + image, selected, np.zeros(11), cols, height=3, chunk_size=chunk, + ) + np.testing.assert_allclose(median_only, med, equal_nan=True) + for normalize in (False, True): + out = si.finalize_activity_image( + image.copy(), selected, np.zeros(selected.size), + np.zeros(11), cols, normalize_mad=normalize, phase_window_height=3, + ) + expected = (9999 - float(med[0, 1, 5])) + if normalize: + expected /= float(mad[0, 1, 5]) + np.testing.assert_allclose(out[0, 1, 5], expected, rtol=1e-6) + self.assertTrue(np.isnan(out[0, 2, 5])) + + def test_empty_fallback_is_nan_without_warnings(self): + import warnings + + image = np.ones((1, 3, 3)) + with warnings.catch_warnings(): + warnings.simplefilter("error", RuntimeWarning) + med, mad = si.phase_matched_nanmedian( + image, np.arange(image.size), np.ones(3), np.arange(3), + return_mad=True, + ) + self.assertTrue(np.isnan(med).all()) + self.assertTrue(np.isnan(mad).all()) + + def test_missing_depth_geometry_and_empty_support(self): + image = np.ones((2, 9, 11)) + cols, centers = np.arange(11), np.full(11, 4) + selected = np.arange(image.size) + with self.assertRaisesRegex(ValueError, "ref_d"): + si.phase_matched_nanmedian(image, selected, centers, cols) + med, mad = si.phase_matched_nanmedian( + image, selected, centers, cols, np.zeros(11), return_mad=True + ) + self.assertTrue(np.isfinite(med[0]).all()) + self.assertTrue(np.isnan(med[1]).all()) + for valid in (np.array([], dtype=int), selected): + med, mad = si.phase_matched_nanmedian( + image * np.nan, valid, centers, cols, np.zeros(11), return_mad=True + ) + self.assertTrue(np.isnan(med).all()) + self.assertTrue(np.isnan(mad).all()) + + def test_finalization_masks_samples_and_preserves_motion_geometry(self): + cols = np.arange(3, 14) + centers = 5 + abs(cols - 8) // 2 + image = np.arange(20 * 20, dtype=float).reshape(1, 20, 20) + selected = np.arange(image.size) + nan_ct = np.zeros(image.size) + nan_ct[::7] = 1 + dr, dc = 2, -1 + image_masked = image.copy() + image_masked.ravel()[nan_ct > 0.5] = np.nan + med, mad = si.phase_matched_nanmedian( + image_masked, selected[nan_ct <= 0.5], centers + dr, cols + dc, + return_mad=True, + ) + for normalize in (True, False): + expected = image_masked - med + if normalize: + positive = np.isfinite(mad) & (mad > 0) + np.divide(expected, mad, out=expected, where=positive) + expected[~positive] = np.nan + actual = si.finalize_activity_image( + image.copy(), selected, nan_ct, centers + dr, cols + dc, + normalize_mad=normalize, + ) + np.testing.assert_allclose(actual, expected, equal_nan=True) + + def test_invalid_parameters(self): + image = np.ones((1, 5, 7)) + for options in ( + {"height": 4}, {"width": 0}, {"chunk_size": 0}, + {"min_samples": 0}, {"phase_tolerance": -1}, + {"phase_tolerance": np.inf}, {"phase_tolerance": np.nan}, + ): + with self.subTest(options=options), self.assertRaises(ValueError): + si.phase_matched_nanmedian( + image, np.arange(image.size), np.full(7, 2), np.arange(7), + **options, + ) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_bandsilo_pipeline.py b/tests/test_bandsilo_pipeline.py index 47326f3..ce20dd4 100644 --- a/tests/test_bandsilo_pipeline.py +++ b/tests/test_bandsilo_pipeline.py @@ -11,6 +11,8 @@ import os import tempfile import unittest +from contextlib import ExitStack +from unittest.mock import patch import h5py import numpy as np @@ -77,6 +79,71 @@ def _path_result( ) +class TestLocalizeMedianMotion(unittest.TestCase): + """Check median geometry without running the expensive rho/NMF stages.""" + + def test_dominant_retained_motion_shifts_only_median_geometry(self): + ref_r = np.array([10, 19]) + ref_c = np.array([8, 8]) + # Dropped frames outnumber either retained bin and must be ignored. + mot_inds = np.array([-1, -1, -1, -1, 0, 1, 1, 1]) + self.assertTrue(pl.SiloParams().normalize_activity_mad) + cases = [ + (shift, normalize) + for shift in ((2, -5), (-3, 4), (0, 0)) + for normalize in (True, False) + ] + for displacement, normalize in cases: + with self.subTest(displacement=displacement, normalize=normalize), ExitStack() as stack: + umyx = np.array([[7, 9], displacement], dtype=float) + for name in ( + "build_motion_h_matrices", "gaussian_kernel_2d", + "selected_pixels_2d_for_plane", "build_convolution_matrix", + "compute_rho", "mask_high_nan_rho", "decay_kernel_1d", + "smooth_rho", + ): + stack.enter_context(patch.object( + pl.bg, name, return_value=(None, None) + )) + stack.enter_context(patch.object( + pl.si, "accumulate_activity_image" + )) + # Stop at finalization; peak detection and NMF are unrelated. + finalize = stack.enter_context(patch.object( + pl.si, "finalize_activity_image", + side_effect=RuntimeError("median reached"), + )) + with self.assertRaisesRegex(RuntimeError, "median reached"): + pl._localize( + residual=np.zeros((2, mot_inds.size)), + umyx=umyx, mot_inds_yx=mot_inds, + sel_pix_idxs=np.array([0]), pixel_coords=None, + n_pixels=1600, ref_d=np.array([0, 0]), + ref_r=ref_r, ref_c=ref_c, num_fast_zs=1, + dmd_pixels_per_column=40, dmd_pixels_per_row=40, + psf2d=np.ones((3, 3)), num_super_pixels=2, + sparse_h_inds=None, sparse_h_vals=None, align_hz=10, + params=pl.SiloParams( + verbose=False, normalize_activity_mad=normalize + ), + ) + finalize.assert_called_once() + self.assertEqual( + finalize.call_args.kwargs["normalize_mad"], normalize + ) + np.testing.assert_array_equal( + finalize.call_args.kwargs["ref_d"], [0, 0] + ) + np.testing.assert_array_equal( + finalize.call_args.kwargs["ref_r"], ref_r + displacement[0] + ) + np.testing.assert_array_equal( + finalize.call_args.kwargs["ref_c"], ref_c + displacement[1] + ) + np.testing.assert_array_equal(ref_r, [10, 19]) + np.testing.assert_array_equal(ref_c, [8, 8]) + + class TestDmdUserRois(unittest.TestCase): """_dmd_user_rois slices a resolved session selection per DMD.""" diff --git a/tests/test_bandsilo_summary_images.py b/tests/test_bandsilo_summary_images.py index 478849b..c7974f0 100644 --- a/tests/test_bandsilo_summary_images.py +++ b/tests/test_bandsilo_summary_images.py @@ -5,6 +5,7 @@ """ import unittest +from unittest.mock import patch import numpy as np @@ -133,32 +134,231 @@ def test_too_few_frames_yields_zero_image(self): class TestFinalizeActivityImage(unittest.TestCase): - """finalize_activity_image masking and local median subtraction.""" + """Activity-image masking, local median subtraction, and MAD scaling.""" - def test_masks_and_subtracts_median(self): - """Never-valid pixels become NaN; valid ones median-subtracted.""" + def test_skip_mad_preserves_median_subtraction_and_mask(self): + image = np.ones((1, 13, 17), dtype=np.float32) + image[0, 6, 8] = 7 + selected = np.arange(1, image.size) + nan_ct = np.zeros(selected.size) + nan_ct[-1] = 0.9 + for geometry in ( + {}, {"ref_r": np.full(17, 6), "ref_c": np.arange(17)}, + ): + with self.subTest(snake=bool(geometry)): + with patch.object( + si, "phase_matched_nanmedian", wraps=si.phase_matched_nanmedian + ) as snake, patch.object( + si, "median_abs_deviation", + side_effect=AssertionError("MAD must not be computed"), + ): + out = si.finalize_activity_image( + image.copy(), selected, nan_ct, + normalize_mad=False, **geometry, + ) + if geometry: + self.assertFalse(snake.call_args.kwargs["return_mad"]) + self.assertEqual(out[0, 6, 8], 6) + self.assertEqual(out[0, 6, 7], 0) + self.assertTrue(np.isnan(out.ravel()[0])) + self.assertTrue(np.isnan(out.ravel()[-1])) + expected = np.zeros_like(image) + expected[0, 6, 8] = 6 + expected.ravel()[[0, -1]] = np.nan + np.testing.assert_allclose(out, expected, equal_nan=True) + + def test_zero_mad_is_masked(self): + """Constant and isolated-spike footprints cannot be normalized.""" num_fast_zs, npc, npr = 1, 13, 13 act_im = np.ones((num_fast_zs, npc, npr), dtype=np.float32) # all pixels selected and valid (nan_ct = 0) sel_pix_idxs = np.arange(npc * npr) nan_ct = np.zeros(sel_pix_idxs.size) - out = si.finalize_activity_image(act_im.copy(), sel_pix_idxs, nan_ct) - # constant image minus its local median -> ~0 everywhere valid - self.assertTrue(np.all(np.isfinite(out))) - np.testing.assert_allclose(out, 0.0, atol=1e-5) + for geometry in ({}, {"ref_r": np.full(npr, 6), "ref_c": np.arange(npr)}): + for spike in (0, 10): + with self.subTest(geometry=bool(geometry), spike=spike): + image = act_im.copy() + image[0, 6, 6] += spike + out = si.finalize_activity_image( + image, sel_pix_idxs, nan_ct, **geometry + ) + self.assertTrue(np.all(np.isnan(out))) def test_high_nan_pixels_masked(self): """Pixels with nan_ct > 0.5 are excluded and set to NaN.""" num_fast_zs, npc, npr = 1, 13, 13 - act_im = np.ones((num_fast_zs, npc, npr), dtype=np.float32) - sel_pix_idxs = np.array([3 * npr + 3, 3 * npr + 4]) - nan_ct = np.array([0.0, 0.9]) # second pixel dropped + act_im = np.arange(npc * npr, dtype=np.float32).reshape(1, npc, npr) + rows, cols = np.meshgrid(np.arange(2, 5), np.arange(2, 8), indexing="ij") + sel_pix_idxs = (rows * npr + cols).ravel() + nan_ct = np.zeros(sel_pix_idxs.size) + nan_ct[sel_pix_idxs == 3 * npr + 4] = 0.9 out = si.finalize_activity_image(act_im, sel_pix_idxs, nan_ct) # the dropped pixel and all unselected pixels are NaN self.assertTrue(np.isnan(out[0, 3, 4])) self.assertTrue(np.isnan(out[0, 0, 0])) self.assertFalse(np.isnan(out[0, 3, 3])) + def test_median_follows_curved_snake(self): + """The 3x11 footprint follows reference-center row changes.""" + size = 17 + cols = np.arange(size) + center_rows = np.abs(cols - size // 2) + 3 + act_im = np.zeros((1, size, size), dtype=np.float32) + for col, row in zip(cols, center_rows): + act_im[0, row - 1 : row + 2, col] = 10.0 + + sel_pix_idxs = np.arange(size * size) + med = si.snake_aligned_nanmedian( + act_im, + sel_pix_idxs, + ref_r=center_rows, + ref_c=cols, + height=3, + ) + + center = size // 2 + self.assertAlmostEqual(float(med[0, 3, center]), 10.0, places=5) + + def test_median_follows_motion_shifted_bend(self): + """Shifted geometry follows the bend in output-image coordinates.""" + cols = np.arange(8, 25) + center_rows = np.abs(cols - 16) + 8 + for dr, dc in ((2, 5), (-2, -5), (0, 0)): + with self.subTest(dr=dr, dc=dc): + act_im = np.zeros((1, 40, 40), dtype=np.float32) + for col, row in zip(cols + dc, center_rows + dr): + act_im[0, row - 1 : row + 2, col] = 10.0 + sel_pix_idxs = np.arange(act_im.size) + med = si.snake_aligned_nanmedian( + act_im, sel_pix_idxs, + ref_r=center_rows + dr, ref_c=cols + dc, + ) + self.assertAlmostEqual(float(med[0, 8 + dr, 16 + dc]), 10.0) + if dc: + unshifted = si.snake_aligned_nanmedian( + act_im, sel_pix_idxs, + ref_r=center_rows, ref_c=cols, + ) + self.assertLess( + float(unshifted[0, 8 + dr, 16 + dc]), 10.0 + ) + + def test_trajectory_does_not_jump_when_outer_snake_is_missing(self): + """Changing snake count does not move the shared trajectory.""" + size = 40 + cols = np.arange(11) + ref_c = np.repeat(cols, 5) + ref_r = np.tile(np.arange(1, 38, 9), cols.size) + keep = ~((ref_c >= 5) & (ref_r == 1)) + + act_im = np.zeros((1, size, cols.size), dtype=np.float32) + act_im[0, 18:21, :] = 10.0 + sel_pix_idxs = np.arange(act_im.size) + med = si.snake_aligned_nanmedian( + act_im, + sel_pix_idxs, + ref_r=ref_r[keep], + ref_c=ref_c[keep], + ) + + self.assertAlmostEqual(float(med[0, 19, 4]), 10.0, places=5) + + def test_raw_mad_normalization(self): + """Both paths use the footprint's own median and unscaled MAD.""" + image = np.broadcast_to(np.arange(17), (1, 17, 17)).astype(np.float32).copy() + image[0, 8, 8] += 6 + selected = np.arange(image.size) + for geometry, height in ( + ({}, 5), + ({"ref_r": np.full(17, 8), "ref_c": np.arange(17)}, 1), + ): + with self.subTest(height=height): + samples = image[0, 8 - height // 2 : 9 + height // 2, 3:14] + median = np.nanmedian(samples) + mad = np.nanmedian(np.abs(samples - median)) + self.assertEqual(mad, 3) + out = si.finalize_activity_image( + image.copy(), selected, np.zeros(selected.size), **geometry + ) + self.assertAlmostEqual( + float(out[0, 8, 8]), float((image[0, 8, 8] - median) / mad) + ) + rescaled = si.finalize_activity_image( + 4 * image + 13, selected, np.zeros(selected.size), **geometry + ) + np.testing.assert_allclose(out, rescaled, equal_nan=True) + + def test_normalization_follows_curved_footprint(self): + cols = np.arange(17) + ref_r = np.abs(cols - 8) + 5 + image = np.zeros((1, 25, 17), dtype=np.float32) + for col, row in zip(cols, ref_r): + image[0, row - 2 : row + 3, col] = 20 + col - 8 + 100 * np.arange(-2, 3) + image[0, 5, 8] = 25 + out = si.finalize_activity_image( + image, np.arange(image.size), np.zeros(image.size), + ref_r=ref_r, ref_c=cols, + ) + # The default 7-row window reaches center-phase samples in columns + # 5:12 at this bend. Replacing 20 with 25 gives median=21, MAD=2. + self.assertAlmostEqual(float(out[0, 5, 8]), 2.0, places=6) + + +class TestSnakeMedianMad(unittest.TestCase): + """MAD shares the exact curved, clipped, section-limited footprint.""" + + def test_matches_explicit_samples_across_chunks(self): + rng = np.random.default_rng(42) + image = rng.normal(size=(2, 20, 17)).astype(np.float32) + image[:, 4, ::3] = np.nan + cols = np.r_[np.arange(7), np.arange(10, 17)] + ref_r = np.abs(cols - 8) + 3 + points = [(0, 3, 0), (0, 4, 6), (1, 10, 10), (1, 19, 16)] + selected = np.ravel_multi_index(np.array(points).T, image.shape) + original = image.copy() + for height in (3, 5): + med, mad = si.snake_aligned_nanmedian( + image, selected, ref_r, cols, height=height, + chunk_size=2, return_mad=True, + ) + for depth, row, col in points: + samples = [] + for sample_col in range(max(0, col - 5), min(17, col + 6)): + # Nearest reference endpoint extends into the gap; + # ties select the left section (columns 0 through 8). + if (sample_col <= 8) != (col <= 8): + continue + nearest_col = cols[np.argmin(np.abs(cols - sample_col))] + delta = abs(nearest_col - 8) - abs(col - 8) + for offset in range(-(height // 2), height // 2 + 1): + sample_row = row + delta + offset + if 0 <= sample_row < image.shape[1]: + samples.append(image[depth, sample_row, sample_col]) + expected_med = np.nanmedian(samples) + expected_mad = np.nanmedian(np.abs(np.array(samples) - expected_med)) + self.assertAlmostEqual(float(med[depth, row, col]), float(expected_med)) + self.assertAlmostEqual(float(mad[depth, row, col]), float(expected_mad)) + unselected = np.ones(image.size, dtype=bool) + unselected[selected] = False + self.assertTrue(np.all(np.isnan(med.ravel()[unselected]))) + self.assertTrue(np.all(np.isnan(mad.ravel()[unselected]))) + median_only = si.snake_aligned_nanmedian( + image, selected, ref_r, cols, height=height + ) + np.testing.assert_allclose(med, median_only, equal_nan=True) + np.testing.assert_array_equal(image, original) + + def test_empty_and_all_nan_support(self): + image = np.full((1, 5, 11), np.nan, dtype=np.float32) + cols = np.arange(11) + for selected in (np.array([], dtype=int), np.array([27])): + with self.subTest(selected=selected.size): + med, mad = si.snake_aligned_nanmedian( + image, selected, np.full(11, 2), cols, return_mad=True + ) + self.assertTrue(np.all(np.isnan(med))) + self.assertTrue(np.all(np.isnan(mad))) + if __name__ == "__main__": unittest.main() From 0bca3be4e3f5a9ebec7a4a50c628d2cbf2b55a7b Mon Sep 17 00:00:00 2001 From: m-xie Date: Wed, 9 Sep 2026 23:45:46 +0000 Subject: [PATCH 29/55] remove edge artifacts --- src/giant_python/bandsilo/summary_images.py | 20 +++++++++- tests/test_bandsilo_phase_filter.py | 7 +++- tests/test_bandsilo_summary_images.py | 44 ++++++++++++++++++++- 3 files changed, 68 insertions(+), 3 deletions(-) diff --git a/src/giant_python/bandsilo/summary_images.py b/src/giant_python/bandsilo/summary_images.py index e146eff..4fc8ed2 100644 --- a/src/giant_python/bandsilo/summary_images.py +++ b/src/giant_python/bandsilo/summary_images.py @@ -270,6 +270,11 @@ def finalize_activity_image( below ``phase_min_samples`` use the mean of finite first-pass medians (and MADs) in the full window, regardless of phase. Empty fallback windows and pixels without reference geometry remain NaN. + Finally, finite input pixels within valid support are set to zero if the + centered vertical window in their own column crosses invalid/nonfinite + input or an image boundary. This overrides undefined median/MAD at these + edges in either normalization mode, without changing the filter samples. + Horizontal window clipping alone does not trigger this suppression. Parameters ---------- @@ -306,7 +311,8 @@ def finalize_activity_image( ndarray Activity in raw local-MAD units (NaN outside the valid support or where the local MAD is zero or non-finite), or median-subtracted - activity in original units when ``normalize_mad=False``. + activity in original units when ``normalize_mad=False``. Valid vertical + support-edge pixels are zero in either mode. """ nan_mask = np.full_like(act_im, True, dtype=bool) valid_sel_pix = np.flatnonzero(nan_ct <= 0.5) @@ -351,13 +357,25 @@ def finalize_activity_image( size=(1, 5, 11), extra_keywords={"nan_policy": "omit"}, ) + # Check original input support, not median/MAD support. The vertical-only + # footprint preserves horizontal edges and never combines depth planes. + valid_input = (~nan_mask) & np.isfinite(act_im) + window_height = ( + phase_window_height if ref_r is not None and ref_c is not None else 5 + ) + vertical_interior = ndimage.minimum_filter1d( + valid_input, size=window_height, axis=1, mode="constant", cval=0, + ) + vertical_edge = valid_input & ~vertical_interior act_im = act_im - med_act_im if not normalize_mad: act_im[nan_mask] = np.nan + act_im[vertical_edge] = 0 return act_im valid_scale = (~nan_mask) & np.isfinite(mad_act_im) & (mad_act_im > 0) np.divide(act_im, mad_act_im, out=act_im, where=valid_scale) act_im[~valid_scale] = np.nan + act_im[vertical_edge] = 0 return act_im diff --git a/tests/test_bandsilo_phase_filter.py b/tests/test_bandsilo_phase_filter.py index b093121..5d1395e 100644 --- a/tests/test_bandsilo_phase_filter.py +++ b/tests/test_bandsilo_phase_filter.py @@ -193,7 +193,7 @@ def test_fallback_uses_first_pass_statistics_not_raw_or_filled_values(self): if normalize: expected /= float(mad[0, 1, 5]) np.testing.assert_allclose(out[0, 1, 5], expected, rtol=1e-6) - self.assertTrue(np.isnan(out[0, 2, 5])) + self.assertEqual(out[0, 2, 5], 0) # Vertical window hits image edge. def test_empty_fallback_is_nan_without_warnings(self): import warnings @@ -246,6 +246,11 @@ def test_finalization_masks_samples_and_preserves_motion_geometry(self): positive = np.isfinite(mad) & (mad > 0) np.divide(expected, mad, out=expected, where=positive) expected[~positive] = np.nan + support = np.isfinite(image_masked) + for z, r, c in zip(*np.where(support)): + if (r < 3 or r + 3 >= image.shape[1] + or not support[z, r - 3:r + 4, c].all()): + expected[z, r, c] = 0 actual = si.finalize_activity_image( image.copy(), selected, nan_ct, centers + dr, cols + dc, normalize_mad=normalize, diff --git a/tests/test_bandsilo_summary_images.py b/tests/test_bandsilo_summary_images.py index c7974f0..70b1e7f 100644 --- a/tests/test_bandsilo_summary_images.py +++ b/tests/test_bandsilo_summary_images.py @@ -136,6 +136,45 @@ def test_too_few_frames_yields_zero_image(self): class TestFinalizeActivityImage(unittest.TestCase): """Activity-image masking, local median subtraction, and MAD scaling.""" + def test_vertical_window_requires_full_valid_support(self): + image = np.ones((2, 19, 13), dtype=np.float32) + selected_mask = np.ones(image.shape, dtype=bool) + selected_mask[0, :4] = False + selected_mask[0, 15:] = False + selected = np.flatnonzero(selected_mask) + nan_ct = np.zeros(selected.size) + dropped = np.ravel_multi_index((0, 9, 6), image.shape) + nan_ct[selected == dropped] = 1 + image[1, 9, 7] = np.nan + image[1, 9, 8] = np.inf + support = selected_mask & np.isfinite(image) + support[0, 9, 6] = False + for height in (1, 3, 7): + geometry = dict(ref_r=np.array([9, 9]), ref_c=np.array([6, 6]), + ref_d=np.array([0, 1]), phase_window_height=height) + for normalize in (False, True): + # Fixed statistics isolate final edge suppression. + stats = ( + (np.zeros_like(image), np.ones_like(image)) + if normalize else np.zeros_like(image) + ) + with patch.object(si, "phase_matched_nanmedian", return_value=stats): + out = si.finalize_activity_image( + image.copy(), selected, nan_ct, normalize_mad=normalize, + **geometry, + ) + for z, r, c in zip(*np.where(support)): + lo, hi = r - height // 2, r + height // 2 + 1 + fits = ( + lo >= 0 and hi <= image.shape[1] + and support[z, lo:hi, c].all() + ) + self.assertEqual(out[z, r, c], 1 if fits else 0) + self.assertTrue(np.isnan(out[~selected_mask]).all()) + self.assertTrue(np.isnan(out[0, 9, 6])) + self.assertTrue(np.isnan(out[1, 9, 7])) + self.assertEqual(out[1, 9, 0], 1) # Horizontal edges survive. + def test_skip_mad_preserves_median_subtraction_and_mask(self): image = np.ones((1, 13, 17), dtype=np.float32) image[0, 6, 8] = 7 @@ -182,7 +221,10 @@ def test_zero_mad_is_masked(self): out = si.finalize_activity_image( image, sel_pix_idxs, nan_ct, **geometry ) - self.assertTrue(np.all(np.isnan(out))) + radius = 3 if geometry else 2 + self.assertTrue(np.all(out[:, :radius] == 0)) + self.assertTrue(np.all(out[:, -radius:] == 0)) + self.assertTrue(np.all(np.isnan(out[:, radius:-radius]))) def test_high_nan_pixels_masked(self): """Pixels with nan_ct > 0.5 are excluded and set to NaN.""" From ad8d61faf7b9193dd265d57b48cd7de8e7e352e9 Mon Sep 17 00:00:00 2001 From: m-xie Date: Wed, 9 Sep 2026 23:57:08 +0000 Subject: [PATCH 30/55] remove mad normalization of activity image --- src/giant_python/bandsilo/pipeline.py | 1 - src/giant_python/bandsilo/summary_images.py | 121 ++++++-------------- src/giant_python/models/params.py | 5 - tests/test_bandsilo_phase_filter.py | 120 ++++++++----------- tests/test_bandsilo_pipeline.py | 17 +-- tests/test_bandsilo_summary_images.py | 107 ++++++++--------- 6 files changed, 128 insertions(+), 243 deletions(-) diff --git a/src/giant_python/bandsilo/pipeline.py b/src/giant_python/bandsilo/pipeline.py index fc46b86..a6a23e6 100644 --- a/src/giant_python/bandsilo/pipeline.py +++ b/src/giant_python/bandsilo/pipeline.py @@ -1002,7 +1002,6 @@ def _localize( nan_ct, ref_r=ref_r + int(dominant_motion[0]), ref_c=ref_c + int(dominant_motion[1]), - normalize_mad=params.normalize_activity_mad, ref_d=ref_d, ) diff --git a/src/giant_python/bandsilo/summary_images.py b/src/giant_python/bandsilo/summary_images.py index 4fc8ed2..67eb4e0 100644 --- a/src/giant_python/bandsilo/summary_images.py +++ b/src/giant_python/bandsilo/summary_images.py @@ -14,7 +14,7 @@ spatio-temporal local maxima (a voxel greater than its 6 spatial neighbors in the same frame and its temporal neighbors), and accumulating their squared value; it is then masked where data was never valid, has a local NaN-median -subtracted, and is divided by the raw local median absolute deviation (MAD). +subtracted, and is zeroed at vertical support edges. The reference's optional ``profile_activity_map`` timing instrumentation is intentionally dropped (it only prints timings and does not affect output). @@ -24,7 +24,6 @@ import numpy as np import scipy.ndimage as ndimage -from scipy.stats import median_abs_deviation from .progress import progress from .trial_data import fast_dilation @@ -247,39 +246,34 @@ def finalize_activity_image( nan_ct: np.ndarray, ref_r: np.ndarray = None, ref_c: np.ndarray = None, - normalize_mad: bool = True, ref_d: np.ndarray = None, phase_window_height: int = 7, phase_window_width: int = 11, phase_tolerance: float = 0.0, phase_min_samples: int = 5, ) -> np.ndarray: - """Mask never-valid pixels and normalize by local NaN-median and MAD. + """Subtract the local NaN-median and suppress vertical support edges. Pixels whose rho was mostly NaN (``nan_ct > 0.5``) or that were never selected are set to NaN. When superpixel reference centers are supplied, an axis-aligned window includes only valid pixels with similar signed vertical offsets from their nearest same-column reference center. No snake identity or shared trajectory is assumed. Without reference - centers, a fixed 5x11 footprint is used. First-pass statistics use - the same local samples: ``MAD = nanmedian(abs(samples - median))``. - The output is ``(act_im - median) / MAD``, with no Gaussian scale factor. - Pixels with zero or non-finite MAD are set to NaN, as are masked pixels. - With ``normalize_mad=False``, only the median is subtracted; MAD is not - computed and zero-MAD pixels remain valid. With geometry, neighborhoods + centers, a fixed 5x11 footprint is used. The output is ``act_im - median`` + in the original activity units. With geometry, neighborhoods below ``phase_min_samples`` use the mean of finite first-pass medians - (and MADs) in the full window, regardless of phase. Empty fallback + in the full window, regardless of phase. Empty fallback windows and pixels without reference geometry remain NaN. Finally, finite input pixels within valid support are set to zero if the centered vertical window in their own column crosses invalid/nonfinite - input or an image boundary. This overrides undefined median/MAD at these - edges in either normalization mode, without changing the filter samples. + input or an image boundary. This overrides undefined medians at these + edges without changing the filter samples. Horizontal window clipping alone does not trigger this suppression. Parameters ---------- act_im : ndarray - Accumulated activity image (masked in place before normalization). + Accumulated activity image (masked in place before subtraction). sel_pix_idxs : ndarray of int Flattened selected-pixel indices into ``act_im``'s grid. nan_ct : ndarray of shape (n_sel,) @@ -290,10 +284,6 @@ def finalize_activity_image( the activity-image coordinate frame. The pipeline shifts these by the dominant retained motion bin as an approximation for the multi-motion image. Multiple bands can occupy the same column. - normalize_mad : bool - Divide the median-subtracted image by raw local MAD (default True). - False skips MAD computation and normalization, retaining masking and - median subtraction. ref_d : ndarray, optional Depth of each reference center. Required with geometry for multi-plane images; omitted centers belong to plane zero in single-plane images. @@ -303,16 +293,15 @@ def finalize_activity_image( Maximum signed-phase difference, in row pixels (default 0, exact equality). Midpoint ties choose the smaller reference row. phase_min_samples : int - Minimum finite phase-matched samples for first-pass median/MAD + Minimum finite phase-matched samples for first-pass medians (default 5); smaller neighborhoods use the full-window fallback. Returns ------- ndarray - Activity in raw local-MAD units (NaN outside the valid support or - where the local MAD is zero or non-finite), or median-subtracted - activity in original units when ``normalize_mad=False``. Valid vertical - support-edge pixels are zero in either mode. + Median-subtracted activity in original units, NaN outside valid + support or where the median is undefined. Valid vertical support-edge + pixels are zero. """ nan_mask = np.full_like(act_im, True, dtype=bool) valid_sel_pix = np.flatnonzero(nan_ct <= 0.5) @@ -324,12 +313,11 @@ def finalize_activity_image( if ref_r is not None and ref_c is not None: # Legacy trajectory-following filter: uncomment this call and comment # out the phase-matched call below to switch back. The original helper - # is retained unchanged below for comparison and regression tests. - # local_stats = snake_aligned_nanmedian( + # is retained below for comparison and regression tests. + # med_act_im = snake_aligned_nanmedian( # act_im, np.flatnonzero(~nan_mask), ref_r, ref_c, - # return_mad=normalize_mad, # ) - local_stats = phase_matched_nanmedian( + med_act_im = phase_matched_nanmedian( act_im, np.flatnonzero(~nan_mask), ref_r, @@ -339,25 +327,12 @@ def finalize_activity_image( width=phase_window_width, phase_tolerance=phase_tolerance, min_samples=phase_min_samples, - return_mad=normalize_mad, ) - if normalize_mad: - med_act_im, mad_act_im = local_stats - else: - med_act_im = local_stats else: med_act_im = ndimage.generic_filter( act_im, np.nanmedian, size=(1, 5, 11) ) - if normalize_mad: - # Use each footprint's own median, not the spatial median image. - mad_act_im = ndimage.generic_filter( - act_im, - median_abs_deviation, - size=(1, 5, 11), - extra_keywords={"nan_policy": "omit"}, - ) - # Check original input support, not median/MAD support. The vertical-only + # Check original input support, not median support. The vertical-only # footprint preserves horizontal edges and never combines depth planes. valid_input = (~nan_mask) & np.isfinite(act_im) window_height = ( @@ -368,13 +343,7 @@ def finalize_activity_image( ) vertical_edge = valid_input & ~vertical_interior act_im = act_im - med_act_im - if not normalize_mad: - act_im[nan_mask] = np.nan - act_im[vertical_edge] = 0 - return act_im - valid_scale = (~nan_mask) & np.isfinite(mad_act_im) & (mad_act_im > 0) - np.divide(act_im, mad_act_im, out=act_im, where=valid_scale) - act_im[~valid_scale] = np.nan + act_im[nan_mask] = np.nan act_im[vertical_edge] = 0 return act_im @@ -390,9 +359,8 @@ def phase_matched_nanmedian( phase_tolerance: float = 0.0, min_samples: int = 5, chunk_size: int = 4096, - return_mad: bool = False, -) -> np.ndarray | tuple[np.ndarray, np.ndarray]: - """Local median/MAD conditioned on signed reference-band phase. +) -> np.ndarray: + """Local median conditioned on signed reference-band phase. For each selected pixel, phase is ``row - nearest_reference_row`` using only reference centers in the same column and depth. Distances are @@ -419,10 +387,7 @@ def phase_matched_nanmedian( This is a phase-conditioned spatial background, not a reconstruction or separation of contributions at physical crossings. - With ``return_mad=True``, first-pass raw MAD uses exactly the median's - samples, without Gaussian scaling. Fallback MAD is the nanmean of finite - first-pass MADs in the full window, not a pooled-sample MAD. Return both - images; otherwise skip MAD entirely. Non-selected/nonfinite pixels and + Non-selected/nonfinite pixels and pixels without geometry remain NaN. Sparse phase/support arrays and chunked gathers bound working memory; only output images are dense. """ @@ -464,9 +429,8 @@ def phase_matched_nanmedian( raise ValueError("valid_pix_idxs contains an out-of-bounds index") selected = np.unique(selected.astype(np.intp)) result = np.full(image.shape, np.nan, dtype=np.result_type(image.dtype, np.float32)) - mad_result = np.full_like(result, np.nan) if return_mad else None if not selected.size: - return (result, mad_result) if return_mad else result + return result nz, nr, nc = image.shape in_bounds = (ref_d >= 0) & (ref_d < nz) & (ref_c >= 0) & (ref_c < nc) @@ -529,28 +493,19 @@ def windows(output_positions): medians = np.nanmedian(samples, axis=1) flat = selected[positions[enough]] result.ravel()[flat] = medians - if return_mad: - np.subtract(samples, medians[:, None], out=samples) - np.abs(samples, out=samples) - mad_result.ravel()[flat] = np.nanmedian(samples, axis=1) # Advanced indexing freezes first-pass values: fallback estimates must # not propagate or depend on chunk size / iteration order. first_medians = result.ravel()[selected] - first_mads = mad_result.ravel()[selected] if return_mad else None missing = outputs[~np.isfinite(first_medians[outputs])] - statistics = [(first_medians, result)] - if return_mad: - statistics.append((first_mads, mad_result)) for positions, lookup, support in windows(missing): - for source, destination in statistics: - accept = support & np.isfinite(source[lookup]) - counts = np.count_nonzero(accept, axis=1) - enough = counts > 0 - # Equivalent to nanmean, without empty-window warnings. - totals = np.sum(np.where(accept, source[lookup], 0), axis=1, dtype=np.float64) - destination.ravel()[selected[positions[enough]]] = totals[enough] / counts[enough] - return (result, mad_result) if return_mad else result + accept = support & np.isfinite(first_medians[lookup]) + counts = np.count_nonzero(accept, axis=1) + enough = counts > 0 + # Equivalent to nanmean, without empty-window warnings. + totals = np.sum(np.where(accept, first_medians[lookup], 0), axis=1, dtype=np.float64) + result.ravel()[selected[positions[enough]]] = totals[enough] / counts[enough] + return result def snake_aligned_nanmedian( @@ -561,9 +516,8 @@ def snake_aligned_nanmedian( height: int = 5, width: int = 11, chunk_size: int = 100_000, - return_mad: bool = False, -) -> np.ndarray | tuple[np.ndarray, np.ndarray]: - """Return local medians and optionally MADs along the band trajectory. +) -> np.ndarray: + """Return local medians along the band trajectory. The parallel snakes have the same trajectory and differ by a nearly fixed vertical spacing. The trajectory is unwrapped modulo that spacing so its @@ -574,12 +528,7 @@ def snake_aligned_nanmedian( ``ref_r`` and ``ref_c`` must already be expressed in ``image`` coordinates; this helper does not apply motion correction itself. - With ``return_mad=True``, return ``(median_image, mad_image)``. The raw - MAD is ``nanmedian(abs(samples - median))`` over exactly the same samples - used for each output median, without Gaussian scaling. The sample buffer - is reused for absolute deviations. Both images are NaN at unrequested - pixels and all-NaN footprints; constant footprints have MAD zero. - The default returns only the median image for compatibility. + The output is NaN at unrequested pixels and all-NaN footprints. """ if height < 1 or width < 1 or height % 2 == 0 or width % 2 == 0: raise ValueError("height and width must be positive odd integers") @@ -639,7 +588,6 @@ def snake_aligned_nanmedian( trajectory_segment = segment_at_reference[nearest] result = np.full_like(image, np.nan, dtype=np.result_type(image, np.float32)) - mad_result = np.full_like(result, np.nan) if return_mad else None col_offsets = np.arange(-(width // 2), width // 2 + 1) row_offsets = np.arange(-(height // 2), height // 2 + 1) @@ -676,9 +624,4 @@ def snake_aligned_nanmedian( ] medians = np.nanmedian(samples, axis=(1, 2)) result.ravel()[flat] = medians - if return_mad: - np.subtract(samples, medians[:, None, None], out=samples) - np.abs(samples, out=samples) - mad_result.ravel()[flat] = np.nanmedian(samples, axis=(1, 2)) - - return (result, mad_result) if return_mad else result + return result diff --git a/src/giant_python/models/params.py b/src/giant_python/models/params.py index 6ac2858..0eb3861 100644 --- a/src/giant_python/models/params.py +++ b/src/giant_python/models/params.py @@ -140,10 +140,6 @@ class SiloParams: DMD (both the low-res read and high-res trace extraction). ``None`` (the default) processes all trials. Use a small value (e.g. ``5``) to iterate quickly while debugging. - normalize_activity_mad : bool - Band-scan activity images are normalized by raw local MAD by default. - False retains local median subtraction but skips MAD computation and - division. """ microscope: str = "slap2" @@ -172,7 +168,6 @@ class SiloParams: operator: str = "SLAP2 User" verbose: bool = False max_trials: Optional[int] = None - normalize_activity_mad: bool = True def set_params( diff --git a/tests/test_bandsilo_phase_filter.py b/tests/test_bandsilo_phase_filter.py index 5d1395e..30c6a4d 100644 --- a/tests/test_bandsilo_phase_filter.py +++ b/tests/test_bandsilo_phase_filter.py @@ -1,4 +1,4 @@ -"""Phase-matched background statistics, including overlapping band columns.""" +"""Phase-matched background medians, including overlapping band columns.""" import unittest @@ -7,12 +7,11 @@ from giant_python.bandsilo import summary_images as si -def brute_stats(image, selected, ref_r, ref_c, ref_d, height, width, - tolerance, minimum): +def brute_median(image, selected, ref_r, ref_c, ref_d, height, width, + tolerance, minimum): """Independent small-grid reference using explicit distance/window loops.""" phases = np.full(image.shape, np.nan) med = np.full(image.shape, np.nan) - mad = np.full(image.shape, np.nan) for flat in selected: z, r, c = np.unravel_index(flat, image.shape) centers = np.unique(ref_r[(ref_d == z) & (ref_c == c)]) @@ -31,20 +30,18 @@ def brute_stats(image, selected, ref_r, ref_c, ref_d, height, width, samples.append(image[z, sr, sc]) if len(samples) >= minimum: med[z, r, c] = np.median(samples) - mad[z, r, c] = np.median(np.abs(samples - med[z, r, c])) - first_med, first_mad = med.copy(), mad.copy() + first_med = med.copy() for flat in selected: z, r, c = np.unravel_index(flat, image.shape) if (not np.isfinite(phases[z, r, c]) or not np.isfinite(image[z, r, c]) or np.isfinite(first_med[z, r, c])): continue - for source, target in ((first_med, med), (first_mad, mad)): - window = source[z, max(0, r - height // 2):r + height // 2 + 1, - max(0, c - width // 2):c + width // 2 + 1] - finite = window[np.isfinite(window)] - if finite.size: - target[z, r, c] = np.mean(finite) - return med, mad + window = first_med[z, max(0, r - height // 2):r + height // 2 + 1, + max(0, c - width // 2):c + width // 2 + 1] + finite = window[np.isfinite(window)] + if finite.size: + med[z, r, c] = np.mean(finite) + return med class TestPhaseMatchedFilter(unittest.TestCase): @@ -63,16 +60,15 @@ def test_overlapping_curved_bands_match_brute_force(self): image.ravel()[selected[::29]] = np.nan image.ravel()[selected[::41]] = np.inf original = image.copy() - expected = brute_stats(image, selected, ref_r, ref_c, ref_d, 9, 7, 0, 3) + expected = brute_median(image, selected, ref_r, ref_c, ref_d, 9, 7, 0, 3) for chunk_size in (1, 53, 4096): actual = si.phase_matched_nanmedian( image, selected[::-1], ref_r, ref_c, ref_d, height=9, width=7, min_samples=3, - chunk_size=chunk_size, return_mad=True, + chunk_size=chunk_size, ) - for values, truth in zip(actual, expected): - np.testing.assert_allclose(values, truth, atol=1e-6, equal_nan=True) - self.assertTrue(np.isnan(values[:, :, 7]).all()) + np.testing.assert_allclose(actual, expected, atol=1e-6, equal_nan=True) + self.assertTrue(np.isnan(actual[:, :, 7]).all()) np.testing.assert_array_equal(image, original) def test_signed_phase_not_absolute_distance(self): @@ -82,12 +78,9 @@ def test_signed_phase_not_absolute_distance(self): image[0, 5, :] = 10 # phase -1 image[0, 7, :] = 100 # phase +1 selected = np.arange(image.size) - med, mad = si.phase_matched_nanmedian( - image, selected, centers, cols, return_mad=True - ) + med = si.phase_matched_nanmedian(image, selected, centers, cols) self.assertEqual(med[0, 5, 5], 10) self.assertEqual(med[0, 7, 5], 100) - self.assertEqual(mad[0, 5, 5], 0) def test_phase_tolerance_and_minimum_samples(self): image = np.arange(5 * 7, dtype=float).reshape(1, 5, 7) @@ -96,15 +89,13 @@ def test_phase_tolerance_and_minimum_samples(self): selected = np.arange(image.size) for tolerance in (0, 0.3, 1.0): for minimum in (1, 5, 100): - truth = brute_stats(image, selected, centers, cols, np.zeros(7), - 5, 7, tolerance, minimum) + truth = brute_median(image, selected, centers, cols, np.zeros(7), + 5, 7, tolerance, minimum) result = si.phase_matched_nanmedian( image, selected, centers, cols, height=5, width=7, phase_tolerance=tolerance, min_samples=minimum, - return_mad=True, ) - for actual, expected in zip(result, truth): - np.testing.assert_allclose(actual, expected, equal_nan=True) + np.testing.assert_allclose(result, truth, equal_nan=True) def test_midpoint_rows_and_duplicate_centers_are_retained(self): image = np.ones((1, 16, 11)) @@ -155,45 +146,38 @@ def test_irregular_spacing_keeps_pixel_offsets_not_fractional_phase(self): ) self.assertEqual(med[0, 3, 3], 4) - def test_fallback_uses_first_pass_statistics_not_raw_or_filled_values(self): + def test_fallback_uses_first_pass_medians_not_raw_or_filled_values(self): image = np.full((1, 3, 11), np.nan, dtype=np.float32) image[0, 0] = np.arange(11) ** 2 image[0, 1:3, 5] = 9999 selected = np.flatnonzero(np.isfinite(image)) cols = np.arange(11) - expected_med, expected_mad = [], [] + expected_med = [] for col in cols: samples = image[0, 0, max(0, col - 5):col + 6] median = np.median(samples) expected_med.append(median) - expected_mad.append(np.median(abs(samples - median))) for chunk in (1, 4, 4096): - med, mad = si.phase_matched_nanmedian( + med = si.phase_matched_nanmedian( image, selected[::-1], np.zeros(11), cols, - height=3, chunk_size=chunk, return_mad=True, + height=3, chunk_size=chunk, ) np.testing.assert_allclose(med[0, 0], expected_med) - np.testing.assert_allclose(mad[0, 0], expected_mad) self.assertAlmostEqual(float(med[0, 1, 5]), np.mean(expected_med), places=5) - self.assertAlmostEqual(float(mad[0, 1, 5]), np.mean(expected_mad), places=5) # Filled row 1 cannot propagate into row 2, even in later chunks. self.assertTrue(np.isnan(med[0, 2, 5])) - self.assertTrue(np.isnan(mad[0, 2, 5])) self.assertTrue(np.isnan(med[~np.isfinite(image)]).all()) - median_only = si.phase_matched_nanmedian( + ordered = si.phase_matched_nanmedian( image, selected, np.zeros(11), cols, height=3, chunk_size=chunk, ) - np.testing.assert_allclose(median_only, med, equal_nan=True) - for normalize in (False, True): - out = si.finalize_activity_image( - image.copy(), selected, np.zeros(selected.size), - np.zeros(11), cols, normalize_mad=normalize, phase_window_height=3, - ) - expected = (9999 - float(med[0, 1, 5])) - if normalize: - expected /= float(mad[0, 1, 5]) - np.testing.assert_allclose(out[0, 1, 5], expected, rtol=1e-6) - self.assertEqual(out[0, 2, 5], 0) # Vertical window hits image edge. + np.testing.assert_allclose(ordered, med, equal_nan=True) + out = si.finalize_activity_image( + image.copy(), selected, np.zeros(selected.size), + np.zeros(11), cols, phase_window_height=3, + ) + expected = 9999 - np.mean(expected_med) + np.testing.assert_allclose(out[0, 1, 5], expected, rtol=1e-6) + self.assertEqual(out[0, 2, 5], 0) # Vertical window hits image edge. def test_empty_fallback_is_nan_without_warnings(self): import warnings @@ -201,12 +185,10 @@ def test_empty_fallback_is_nan_without_warnings(self): image = np.ones((1, 3, 3)) with warnings.catch_warnings(): warnings.simplefilter("error", RuntimeWarning) - med, mad = si.phase_matched_nanmedian( + med = si.phase_matched_nanmedian( image, np.arange(image.size), np.ones(3), np.arange(3), - return_mad=True, ) self.assertTrue(np.isnan(med).all()) - self.assertTrue(np.isnan(mad).all()) def test_missing_depth_geometry_and_empty_support(self): image = np.ones((2, 9, 11)) @@ -214,17 +196,16 @@ def test_missing_depth_geometry_and_empty_support(self): selected = np.arange(image.size) with self.assertRaisesRegex(ValueError, "ref_d"): si.phase_matched_nanmedian(image, selected, centers, cols) - med, mad = si.phase_matched_nanmedian( - image, selected, centers, cols, np.zeros(11), return_mad=True + med = si.phase_matched_nanmedian( + image, selected, centers, cols, np.zeros(11) ) self.assertTrue(np.isfinite(med[0]).all()) self.assertTrue(np.isnan(med[1]).all()) for valid in (np.array([], dtype=int), selected): - med, mad = si.phase_matched_nanmedian( - image * np.nan, valid, centers, cols, np.zeros(11), return_mad=True + med = si.phase_matched_nanmedian( + image * np.nan, valid, centers, cols, np.zeros(11) ) self.assertTrue(np.isnan(med).all()) - self.assertTrue(np.isnan(mad).all()) def test_finalization_masks_samples_and_preserves_motion_geometry(self): cols = np.arange(3, 14) @@ -236,26 +217,19 @@ def test_finalization_masks_samples_and_preserves_motion_geometry(self): dr, dc = 2, -1 image_masked = image.copy() image_masked.ravel()[nan_ct > 0.5] = np.nan - med, mad = si.phase_matched_nanmedian( + med = si.phase_matched_nanmedian( image_masked, selected[nan_ct <= 0.5], centers + dr, cols + dc, - return_mad=True, ) - for normalize in (True, False): - expected = image_masked - med - if normalize: - positive = np.isfinite(mad) & (mad > 0) - np.divide(expected, mad, out=expected, where=positive) - expected[~positive] = np.nan - support = np.isfinite(image_masked) - for z, r, c in zip(*np.where(support)): - if (r < 3 or r + 3 >= image.shape[1] - or not support[z, r - 3:r + 4, c].all()): - expected[z, r, c] = 0 - actual = si.finalize_activity_image( - image.copy(), selected, nan_ct, centers + dr, cols + dc, - normalize_mad=normalize, - ) - np.testing.assert_allclose(actual, expected, equal_nan=True) + expected = image_masked - med + support = np.isfinite(image_masked) + for z, r, c in zip(*np.where(support)): + if (r < 3 or r + 3 >= image.shape[1] + or not support[z, r - 3:r + 4, c].all()): + expected[z, r, c] = 0 + actual = si.finalize_activity_image( + image.copy(), selected, nan_ct, centers + dr, cols + dc, + ) + np.testing.assert_allclose(actual, expected, equal_nan=True) def test_invalid_parameters(self): image = np.ones((1, 5, 7)) diff --git a/tests/test_bandsilo_pipeline.py b/tests/test_bandsilo_pipeline.py index ce20dd4..a0bc558 100644 --- a/tests/test_bandsilo_pipeline.py +++ b/tests/test_bandsilo_pipeline.py @@ -87,14 +87,8 @@ def test_dominant_retained_motion_shifts_only_median_geometry(self): ref_c = np.array([8, 8]) # Dropped frames outnumber either retained bin and must be ignored. mot_inds = np.array([-1, -1, -1, -1, 0, 1, 1, 1]) - self.assertTrue(pl.SiloParams().normalize_activity_mad) - cases = [ - (shift, normalize) - for shift in ((2, -5), (-3, 4), (0, 0)) - for normalize in (True, False) - ] - for displacement, normalize in cases: - with self.subTest(displacement=displacement, normalize=normalize), ExitStack() as stack: + for displacement in ((2, -5), (-3, 4), (0, 0)): + with self.subTest(displacement=displacement), ExitStack() as stack: umyx = np.array([[7, 9], displacement], dtype=float) for name in ( "build_motion_h_matrices", "gaussian_kernel_2d", @@ -123,14 +117,9 @@ def test_dominant_retained_motion_shifts_only_median_geometry(self): dmd_pixels_per_column=40, dmd_pixels_per_row=40, psf2d=np.ones((3, 3)), num_super_pixels=2, sparse_h_inds=None, sparse_h_vals=None, align_hz=10, - params=pl.SiloParams( - verbose=False, normalize_activity_mad=normalize - ), + params=pl.SiloParams(verbose=False), ) finalize.assert_called_once() - self.assertEqual( - finalize.call_args.kwargs["normalize_mad"], normalize - ) np.testing.assert_array_equal( finalize.call_args.kwargs["ref_d"], [0, 0] ) diff --git a/tests/test_bandsilo_summary_images.py b/tests/test_bandsilo_summary_images.py index 70b1e7f..aa3fec4 100644 --- a/tests/test_bandsilo_summary_images.py +++ b/tests/test_bandsilo_summary_images.py @@ -134,7 +134,7 @@ def test_too_few_frames_yields_zero_image(self): class TestFinalizeActivityImage(unittest.TestCase): - """Activity-image masking, local median subtraction, and MAD scaling.""" + """Activity-image masking and local median subtraction in raw units.""" def test_vertical_window_requires_full_valid_support(self): image = np.ones((2, 19, 13), dtype=np.float32) @@ -152,30 +152,26 @@ def test_vertical_window_requires_full_valid_support(self): for height in (1, 3, 7): geometry = dict(ref_r=np.array([9, 9]), ref_c=np.array([6, 6]), ref_d=np.array([0, 1]), phase_window_height=height) - for normalize in (False, True): - # Fixed statistics isolate final edge suppression. - stats = ( - (np.zeros_like(image), np.ones_like(image)) - if normalize else np.zeros_like(image) + # A fixed median isolates final edge suppression. + with patch.object( + si, "phase_matched_nanmedian", return_value=np.zeros_like(image) + ): + out = si.finalize_activity_image( + image.copy(), selected, nan_ct, **geometry, ) - with patch.object(si, "phase_matched_nanmedian", return_value=stats): - out = si.finalize_activity_image( - image.copy(), selected, nan_ct, normalize_mad=normalize, - **geometry, - ) - for z, r, c in zip(*np.where(support)): - lo, hi = r - height // 2, r + height // 2 + 1 - fits = ( - lo >= 0 and hi <= image.shape[1] - and support[z, lo:hi, c].all() - ) - self.assertEqual(out[z, r, c], 1 if fits else 0) - self.assertTrue(np.isnan(out[~selected_mask]).all()) - self.assertTrue(np.isnan(out[0, 9, 6])) - self.assertTrue(np.isnan(out[1, 9, 7])) - self.assertEqual(out[1, 9, 0], 1) # Horizontal edges survive. + for z, r, c in zip(*np.where(support)): + lo, hi = r - height // 2, r + height // 2 + 1 + fits = ( + lo >= 0 and hi <= image.shape[1] + and support[z, lo:hi, c].all() + ) + self.assertEqual(out[z, r, c], 1 if fits else 0) + self.assertTrue(np.isnan(out[~selected_mask]).all()) + self.assertTrue(np.isnan(out[0, 9, 6])) + self.assertTrue(np.isnan(out[1, 9, 7])) + self.assertEqual(out[1, 9, 0], 1) # Horizontal edges survive. - def test_skip_mad_preserves_median_subtraction_and_mask(self): + def test_median_subtraction_preserves_mask(self): image = np.ones((1, 13, 17), dtype=np.float32) image[0, 6, 8] = 7 selected = np.arange(1, image.size) @@ -185,18 +181,9 @@ def test_skip_mad_preserves_median_subtraction_and_mask(self): {}, {"ref_r": np.full(17, 6), "ref_c": np.arange(17)}, ): with self.subTest(snake=bool(geometry)): - with patch.object( - si, "phase_matched_nanmedian", wraps=si.phase_matched_nanmedian - ) as snake, patch.object( - si, "median_abs_deviation", - side_effect=AssertionError("MAD must not be computed"), - ): - out = si.finalize_activity_image( - image.copy(), selected, nan_ct, - normalize_mad=False, **geometry, - ) - if geometry: - self.assertFalse(snake.call_args.kwargs["return_mad"]) + out = si.finalize_activity_image( + image.copy(), selected, nan_ct, **geometry, + ) self.assertEqual(out[0, 6, 8], 6) self.assertEqual(out[0, 6, 7], 0) self.assertTrue(np.isnan(out.ravel()[0])) @@ -206,8 +193,8 @@ def test_skip_mad_preserves_median_subtraction_and_mask(self): expected.ravel()[[0, -1]] = np.nan np.testing.assert_allclose(out, expected, equal_nan=True) - def test_zero_mad_is_masked(self): - """Constant and isolated-spike footprints cannot be normalized.""" + def test_constant_neighborhoods_are_zero_and_spike_is_preserved(self): + """Subtract constant backgrounds without masking valid neighborhoods.""" num_fast_zs, npc, npr = 1, 13, 13 act_im = np.ones((num_fast_zs, npc, npr), dtype=np.float32) # all pixels selected and valid (nan_ct = 0) @@ -224,7 +211,9 @@ def test_zero_mad_is_masked(self): radius = 3 if geometry else 2 self.assertTrue(np.all(out[:, :radius] == 0)) self.assertTrue(np.all(out[:, -radius:] == 0)) - self.assertTrue(np.all(np.isnan(out[:, radius:-radius]))) + expected = np.zeros_like(act_im) + expected[0, 6, 6] = spike + np.testing.assert_array_equal(out, expected) def test_high_nan_pixels_masked(self): """Pixels with nan_ct > 0.5 are excluded and set to NaN.""" @@ -305,8 +294,8 @@ def test_trajectory_does_not_jump_when_outer_snake_is_missing(self): self.assertAlmostEqual(float(med[0, 19, 4]), 10.0, places=5) - def test_raw_mad_normalization(self): - """Both paths use the footprint's own median and unscaled MAD.""" + def test_median_subtraction_preserves_raw_units_and_scaling(self): + """Both paths subtract their footprint median and retain input scaling.""" image = np.broadcast_to(np.arange(17), (1, 17, 17)).astype(np.float32).copy() image[0, 8, 8] += 6 selected = np.arange(image.size) @@ -317,20 +306,18 @@ def test_raw_mad_normalization(self): with self.subTest(height=height): samples = image[0, 8 - height // 2 : 9 + height // 2, 3:14] median = np.nanmedian(samples) - mad = np.nanmedian(np.abs(samples - median)) - self.assertEqual(mad, 3) out = si.finalize_activity_image( image.copy(), selected, np.zeros(selected.size), **geometry ) self.assertAlmostEqual( - float(out[0, 8, 8]), float((image[0, 8, 8] - median) / mad) + float(out[0, 8, 8]), float(image[0, 8, 8] - median) ) rescaled = si.finalize_activity_image( 4 * image + 13, selected, np.zeros(selected.size), **geometry ) - np.testing.assert_allclose(out, rescaled, equal_nan=True) + np.testing.assert_allclose(4 * out, rescaled, equal_nan=True) - def test_normalization_follows_curved_footprint(self): + def test_median_subtraction_follows_curved_footprint(self): cols = np.arange(17) ref_r = np.abs(cols - 8) + 5 image = np.zeros((1, 25, 17), dtype=np.float32) @@ -342,12 +329,12 @@ def test_normalization_follows_curved_footprint(self): ref_r=ref_r, ref_c=cols, ) # The default 7-row window reaches center-phase samples in columns - # 5:12 at this bend. Replacing 20 with 25 gives median=21, MAD=2. - self.assertAlmostEqual(float(out[0, 5, 8]), 2.0, places=6) + # 5:12 at this bend. Replacing 20 with 25 gives median=21. + self.assertAlmostEqual(float(out[0, 5, 8]), 4.0, places=6) -class TestSnakeMedianMad(unittest.TestCase): - """MAD shares the exact curved, clipped, section-limited footprint.""" +class TestSnakeMedian(unittest.TestCase): + """Medians use the exact curved, clipped, section-limited footprint.""" def test_matches_explicit_samples_across_chunks(self): rng = np.random.default_rng(42) @@ -359,9 +346,9 @@ def test_matches_explicit_samples_across_chunks(self): selected = np.ravel_multi_index(np.array(points).T, image.shape) original = image.copy() for height in (3, 5): - med, mad = si.snake_aligned_nanmedian( + med = si.snake_aligned_nanmedian( image, selected, ref_r, cols, height=height, - chunk_size=2, return_mad=True, + chunk_size=2, ) for depth, row, col in points: samples = [] @@ -377,17 +364,16 @@ def test_matches_explicit_samples_across_chunks(self): if 0 <= sample_row < image.shape[1]: samples.append(image[depth, sample_row, sample_col]) expected_med = np.nanmedian(samples) - expected_mad = np.nanmedian(np.abs(np.array(samples) - expected_med)) self.assertAlmostEqual(float(med[depth, row, col]), float(expected_med)) - self.assertAlmostEqual(float(mad[depth, row, col]), float(expected_mad)) unselected = np.ones(image.size, dtype=bool) unselected[selected] = False self.assertTrue(np.all(np.isnan(med.ravel()[unselected]))) - self.assertTrue(np.all(np.isnan(mad.ravel()[unselected]))) - median_only = si.snake_aligned_nanmedian( - image, selected, ref_r, cols, height=height - ) - np.testing.assert_allclose(med, median_only, equal_nan=True) + for chunk_size in (1, 4096): + other_chunk = si.snake_aligned_nanmedian( + image, selected, ref_r, cols, height=height, + chunk_size=chunk_size, + ) + np.testing.assert_allclose(med, other_chunk, equal_nan=True) np.testing.assert_array_equal(image, original) def test_empty_and_all_nan_support(self): @@ -395,11 +381,10 @@ def test_empty_and_all_nan_support(self): cols = np.arange(11) for selected in (np.array([], dtype=int), np.array([27])): with self.subTest(selected=selected.size): - med, mad = si.snake_aligned_nanmedian( - image, selected, np.full(11, 2), cols, return_mad=True + med = si.snake_aligned_nanmedian( + image, selected, np.full(11, 2), cols ) self.assertTrue(np.all(np.isnan(med))) - self.assertTrue(np.all(np.isnan(mad))) if __name__ == "__main__": From 57e6d83cad32a02a364eea0fb6359e2b34f9e82b Mon Sep 17 00:00:00 2001 From: m-xie Date: Thu, 10 Sep 2026 07:31:15 +0000 Subject: [PATCH 31/55] allow anisotropic peak fitting in activity image --- src/giant_python/bandsilo/peaks.py | 194 ++++++++++++++++++++--------- tests/test_bandsilo_peaks.py | 180 ++++++++++++++++++++++---- 2 files changed, 294 insertions(+), 80 deletions(-) diff --git a/src/giant_python/bandsilo/peaks.py b/src/giant_python/bandsilo/peaks.py index e06fb39..1fda7d6 100644 --- a/src/giant_python/bandsilo/peaks.py +++ b/src/giant_python/bandsilo/peaks.py @@ -7,7 +7,7 @@ :mod:`giant_python.bandsilo.summary_images`; those seeds initialize the Phase-6 NMF localization. -Detection fits integrated isotropic 2-D Gaussians (each Gaussian integrated +Detection fits integrated axis-aligned 2-D Gaussians (each Gaussian integrated over unit pixels) to the activity image with a bounded Levenberg-Marquardt solver (:func:`_lsq_curvefit`, using the analytical Jacobian from :func:`_gaussian_peaks_integrated_val_jac`). Per plane, an initial set of local @@ -34,13 +34,13 @@ _AMP_SCALE = 1.0 / 0.75 # MAD -> Gaussian-sigma scale factor (median absolute deviation / 0.6745). -_MAD_TO_SIGMA = 0.6741891400433162 +_MAD_TO_SIGMA = 0.67449 def gaussian_peaks_integrated( theta: np.ndarray, yxdata: np.ndarray ) -> np.ndarray: - """Evaluate integrated isotropic 2-D Gaussians at selected pixels. + """Evaluate integrated axis-aligned 2-D Gaussians at selected pixels. Each Gaussian is integrated over unit pixels on a regular grid (port of ``gaussianPeaksIntegrated``). Only the ``M`` requested pixels are evaluated @@ -48,8 +48,9 @@ def gaussian_peaks_integrated( Parameters ---------- - theta : ndarray of shape (N, 4) - Per-Gaussian ``[amp, mu_y, mu_x, sigma]``. + theta : ndarray of shape (N, 5) or (N, 4) + Per-Gaussian ``[amp, mu_y, mu_x, sigma_y, sigma_x]``. A four-column + array uses a shared ``sigma`` for both axes (legacy isotropic form). yxdata : ndarray of shape (M, 2) Pixel-center ``[y, x]`` coordinates. @@ -70,7 +71,8 @@ def gaussian_peaks_integrated( amp = theta[:, 0] my = theta[:, 1] mx = theta[:, 2] - s = np.maximum(theta[:, 3], np.finfo(float).eps) + sy = np.maximum(theta[:, 3], np.finfo(float).eps) + sx = np.maximum(theta[:, -1], np.finfo(float).eps) c = np.sqrt(np.pi / 2) rt2 = np.sqrt(2.0) @@ -80,10 +82,10 @@ def gaussian_peaks_integrated( xr = (xc + 0.5)[:, np.newaxis] ix = ( c - * s[np.newaxis, :] + * sx[np.newaxis, :] * ( - erf((xr - mx[np.newaxis, :]) / (rt2 * s[np.newaxis, :])) - - erf((xl - mx[np.newaxis, :]) / (rt2 * s[np.newaxis, :])) + erf((xr - mx[np.newaxis, :]) / (rt2 * sx[np.newaxis, :])) + - erf((xl - mx[np.newaxis, :]) / (rt2 * sx[np.newaxis, :])) ) ) @@ -92,10 +94,10 @@ def gaussian_peaks_integrated( yt = (yc + 0.5)[:, np.newaxis] iy = ( c - * s[np.newaxis, :] + * sy[np.newaxis, :] * ( - erf((yt - my[np.newaxis, :]) / (rt2 * s[np.newaxis, :])) - - erf((yb - my[np.newaxis, :]) / (rt2 * s[np.newaxis, :])) + erf((yt - my[np.newaxis, :]) / (rt2 * sy[np.newaxis, :])) + - erf((yb - my[np.newaxis, :]) / (rt2 * sy[np.newaxis, :])) ) ) @@ -109,12 +111,13 @@ def _gaussian_peaks_integrated_val_jac( Shares the integrated-profile intermediates between the forward value and the Jacobian so the LM solver needs one evaluation per iteration instead of - ``4*N`` finite differences. + ``P*N`` finite differences, where ``P`` is the number of parameters. Parameters ---------- - theta : ndarray of shape (N, 4) - Per-Gaussian ``[amp, mu_y, mu_x, sigma]``. + theta : ndarray of shape (N, 5) or (N, 4) + Per-Gaussian ``[amp, mu_y, mu_x, sigma_y, sigma_x]``, or the legacy + isotropic form ``[amp, mu_y, mu_x, sigma]``. yxdata : ndarray of shape (M, 2) Pixel-center ``[y, x]`` coordinates. @@ -122,10 +125,9 @@ def _gaussian_peaks_integrated_val_jac( ------- val : ndarray of shape (M,) Forward value at each pixel. - jac : ndarray of shape (M, 4*N) - Column-major-per-Gaussian Jacobian (columns ``0::4`` are ``d/dA``, - ``1::4`` are ``d/dmu_y``, ``2::4`` are ``d/dmu_x``, ``3::4`` are - ``d/dsigma``). + jac : ndarray of shape (M, P*N) + Derivatives in the same order as ``theta.ravel()``. The isotropic + form sums the two width derivatives into one shared-sigma derivative. """ x_int = yxdata[:, 1].astype(np.intp) y_int = yxdata[:, 0].astype(np.intp) @@ -142,29 +144,31 @@ def _gaussian_peaks_integrated_val_jac( amp = theta[:, 0] my = theta[:, 1] mx = theta[:, 2] - s = np.maximum(theta[:, 3], np.finfo(float).eps) + sy = np.maximum(theta[:, 3], np.finfo(float).eps) + sx = np.maximum(theta[:, -1], np.finfo(float).eps) c = np.sqrt(np.pi / 2) rt2 = np.sqrt(2.0) - inv_rt2s = 1.0 / (rt2 * s[np.newaxis, :]) + inv_rt2sx = 1.0 / (rt2 * sx[np.newaxis, :]) + inv_rt2sy = 1.0 / (rt2 * sy[np.newaxis, :]) xc = np.arange(x_min, x_max + 1, dtype=float) xl = (xc - 0.5)[:, np.newaxis] xr = (xc + 0.5)[:, np.newaxis] - ux_l = (xl - mx[np.newaxis, :]) * inv_rt2s - ux_r = (xr - mx[np.newaxis, :]) * inv_rt2s + ux_l = (xl - mx[np.newaxis, :]) * inv_rt2sx + ux_r = (xr - mx[np.newaxis, :]) * inv_rt2sx erf_ux_l = erf(ux_l) erf_ux_r = erf(ux_r) - ix = c * s[np.newaxis, :] * (erf_ux_r - erf_ux_l) + ix = c * sx[np.newaxis, :] * (erf_ux_r - erf_ux_l) yc = np.arange(y_min, y_max + 1, dtype=float) yb = (yc - 0.5)[:, np.newaxis] yt = (yc + 0.5)[:, np.newaxis] - uy_b = (yb - my[np.newaxis, :]) * inv_rt2s - uy_t = (yt - my[np.newaxis, :]) * inv_rt2s + uy_b = (yb - my[np.newaxis, :]) * inv_rt2sy + uy_t = (yt - my[np.newaxis, :]) * inv_rt2sy erf_uy_b = erf(uy_b) erf_uy_t = erf(uy_t) - iy = c * s[np.newaxis, :] * (erf_uy_t - erf_uy_b) + iy = c * sy[np.newaxis, :] * (erf_uy_t - erf_uy_b) iy_m = iy[y_idx, :] ix_m = ix[x_idx, :] @@ -184,18 +188,24 @@ def _gaussian_peaks_integrated_val_jac( sqrt2 = rt2 d_iy_ds_m = ( - iy / s[np.newaxis, :] + sqrt2 * (uy_b * exp_uyb2 - uy_t * exp_uyt2) + iy / sy[np.newaxis, :] + sqrt2 * (uy_b * exp_uyb2 - uy_t * exp_uyt2) )[y_idx, :] d_ix_ds_m = ( - ix / s[np.newaxis, :] + sqrt2 * (ux_l * exp_uxl2 - ux_r * exp_uxr2) + ix / sx[np.newaxis, :] + sqrt2 * (ux_l * exp_uxl2 - ux_r * exp_uxr2) )[x_idx, :] - dval_ds = amp[np.newaxis, :] * (d_iy_ds_m * ix_m + iy_m * d_ix_ds_m) - - jac = np.empty((m, 4 * n), dtype=float) - jac[:, 0::4] = dval_da - jac[:, 1::4] = dval_dmy - jac[:, 2::4] = dval_dmx - jac[:, 3::4] = dval_ds + dval_dsy = amp[np.newaxis, :] * d_iy_ds_m * ix_m + dval_dsx = amp[np.newaxis, :] * iy_m * d_ix_ds_m + + n_params = theta.shape[1] + jac = np.empty((m, n_params * n), dtype=float) + jac[:, 0::n_params] = dval_da + jac[:, 1::n_params] = dval_dmy + jac[:, 2::n_params] = dval_dmx + if n_params == 4: + jac[:, 3::n_params] = dval_dsy + dval_dsx + else: + jac[:, 3::n_params] = dval_dsy + jac[:, 4::n_params] = dval_dsx return val, jac @@ -216,25 +226,27 @@ def _lsq_curvefit( Parameters ---------- - theta0 : ndarray of shape (N, 4) - Initial ``[amp, mu_y, mu_x, sigma]`` per Gaussian. + theta0 : ndarray of shape (N, P) + Initial ``[amp, mu_y, mu_x, sigma_y, sigma_x]`` per Gaussian (P=5), + or legacy isotropic parameters (P=4). xdata : ndarray of shape (M, 2) Pixel-center ``[y, x]`` coordinates. ydata : ndarray of shape (M,) Observed values (``act_im(sel) - mu_bg``). - lb_flat, ub_flat : ndarray of shape (4*N,) + lb_flat, ub_flat : ndarray of shape (P*N,) Row-major-flattened lower/upper bounds. max_nfev : int Cap on value+Jacobian evaluations. Returns ------- - ndarray of shape (N, 4) + ndarray of shape (N, P) Optimized parameters. """ + n_params = theta0.shape[1] x = np.clip(theta0.ravel().copy(), lb_flat, ub_flat) - val, jac = _gaussian_peaks_integrated_val_jac(x.reshape(-1, 4), xdata) + val, jac = _gaussian_peaks_integrated_val_jac(x.reshape(-1, n_params), xdata) r = val - ydata cost = np.dot(r, r) nfev = 1 @@ -253,7 +265,7 @@ def _lsq_curvefit( x_new = np.clip(x + delta, lb_flat, ub_flat) val_new, jac_new = _gaussian_peaks_integrated_val_jac( - x_new.reshape(-1, 4), xdata + x_new.reshape(-1, n_params), xdata ) r_new = val_new - ydata cost_new = np.dot(r_new, r_new) @@ -274,7 +286,54 @@ def _lsq_curvefit( else: lam = min(lam * 10.0, 1e10) - return x.reshape(-1, 4) + return x.reshape(-1, n_params) + + +def _fit_isotropic_then_anisotropic( + theta0: np.ndarray, + xdata: np.ndarray, + ydata: np.ndarray, + lb_flat: np.ndarray, + ub_flat: np.ndarray, + max_nfev: int = 5000, +) -> np.ndarray: + """Fit shared widths first, then relax to independent y/x widths. + + All peaks in the supplied fit region are optimized jointly in both stages. + The first stage uses four parameters per peak and constrains its shared + sigma to the intersection of the y/x bounds (normally [0.35, 1] pixels). + The second stage starts from that solution with sigma_y == sigma_x and + restores the original five-parameter bounds. Each stage has its own + ``max_nfev`` budget. + + This is an initialization preference, not an anisotropy penalty: widths + can separate whenever that lowers the residual sum of squares. The LM + solver accepts only improving steps, retaining the isotropic solution + when relaxation cannot improve it. + """ + if theta0.shape[0] == 0: + return theta0.copy() + + lb = lb_flat.reshape(-1, 5) + ub = ub_flat.reshape(-1, 5) + iso_lb = lb[:, :4].copy() + iso_ub = ub[:, :4].copy() + iso_lb[:, 3] = np.maximum(lb[:, 3], lb[:, 4]) + iso_ub[:, 3] = np.minimum(ub[:, 3], ub[:, 4]) + if np.any(iso_lb[:, 3] > iso_ub[:, 3]): + raise ValueError("Isotropic fitting requires overlapping y/x width bounds") + + clipped = np.clip(theta0, lb, ub) + iso0 = clipped[:, :4].copy() + iso0[:, 3] = np.sqrt(clipped[:, 3] * clipped[:, 4]) + isotropic = _lsq_curvefit( + iso0, xdata, ydata, iso_lb.ravel(), iso_ub.ravel(), + max_nfev=max_nfev, + ) + relaxed0 = np.column_stack([isotropic, isotropic[:, 3]]) + return _lsq_curvefit( + relaxed0, xdata, ydata, lb_flat, ub_flat, max_nfev=max_nfev + ) def _make_bounds( @@ -283,7 +342,8 @@ def _make_bounds( """Return flattened LM bounds for peaks located at ``plocs``. Amplitude is unbounded above; means are constrained to +/- 1.5 px of the - seed (clipped to the image); sigma to ``[0.35, 5.0]``. + seed (clipped to the image); sigma_y to ``[0.35, 8.0]`` and sigma_x to + ``[0.35, 1.0]``, in pixels. Parameters ---------- @@ -294,7 +354,7 @@ def _make_bounds( Returns ------- - lb_flat, ub_flat : ndarray of shape (4*N,) + lb_flat, ub_flat : ndarray of shape (5*N,) Row-major-flattened lower/upper bounds. """ n = plocs.shape[0] @@ -304,6 +364,7 @@ def _make_bounds( np.maximum(0, plocs[:, 0] - 1.5), np.maximum(0, plocs[:, 1] - 1.5), np.ones(n) * 0.35, + np.ones(n) * 0.35, ] ) ub = np.column_stack( @@ -311,7 +372,8 @@ def _make_bounds( np.full(n, np.inf), np.minimum(height - 1, plocs[:, 0] + 1.5), np.minimum(width - 1, plocs[:, 1] + 1.5), - np.full(n, 5.0), + np.full(n, 8.0), + np.full(n, 1.0), ] ) return lb.ravel(), ub.ravel() @@ -358,7 +420,7 @@ def _process_cc_new_peak( ---------- new_peak : tuple ``((pY, pX), cc_label)`` for the candidate. - thetaf : ndarray of shape (N, 4) + thetaf : ndarray of shape (N, 5) Current Gaussian parameters. p_locs : ndarray of shape (N, 2) Current seed locations. @@ -383,7 +445,9 @@ def _process_cc_new_peak( amp_new = act_im_2d[py_new, px_new] * _AMP_SCALE n_before = thetaf.shape[0] - thetaf = np.vstack([thetaf, [amp_new, float(py_new), float(px_new), 0.5]]) + thetaf = np.vstack( + [thetaf, [amp_new, float(py_new), float(px_new), 0.5, 0.5]] + ) p_locs = np.vstack([p_locs, [float(py_new), float(px_new)]]) new_idx = n_before @@ -396,7 +460,7 @@ def _process_cc_new_peak( in_cc = cc_mask[iy, ix] lb_cc, ub_cc = _make_bounds(p_locs[in_cc], height, width) - thetaf[in_cc] = _lsq_curvefit( + thetaf[in_cc] = _fit_isotropic_then_anisotropic( thetaf[in_cc], cc_yx, cc_vals, lb_cc, ub_cc, max_nfev=5000 ) @@ -410,7 +474,7 @@ def _process_cc_new_peak( refit_mask[new_idx] = False if np.any(refit_mask): lb_rf, ub_rf = _make_bounds(p_locs[refit_mask], height, width) - thetaf[refit_mask] = _lsq_curvefit( + thetaf[refit_mask] = _fit_isotropic_then_anisotropic( thetaf[refit_mask], cc_yx, cc_vals, lb_rf, ub_rf, max_nfev=5000 ) thetaf = np.delete(thetaf, new_idx, axis=0) @@ -463,7 +527,7 @@ def _refine_residual_peaks( Returns ------- - ndarray of shape (N, 4) + ndarray of shape (N, 5) The refined Gaussian parameters. """ reject_mask = np.zeros((height, width), dtype=bool) @@ -533,6 +597,7 @@ def detect_peaks_2d( Finds initial local maxima (above ``peak_thresh``), fits them jointly, then iteratively adds residual peaks (:func:`_refine_residual_peaks`), and finally drops peaks whose amplitude falls below a sigma-adjusted threshold. + Every fit/refit optimizes shared widths first, then allows anisotropy. Parameters ---------- @@ -552,11 +617,12 @@ def detect_peaks_2d( Returns ------- - ndarray of shape (N, 4) - ``[amp, mu_y, mu_x, sigma]`` per peak, or ``(0, 4)`` if none. + ndarray of shape (N, 5) + ``[amp, mu_y, mu_x, sigma_y, sigma_x]`` per peak, or ``(0, 5)`` if + none. Widths are bounded to ``[0.35, 8]`` in y and ``[0.35, 1]`` in x. """ height, width = act_im_2d.shape - empty = np.zeros((0, 4)) + empty = np.zeros((0, 5)) explored = act_im_2d.copy() explored[exclusion_mask | np.isnan(explored)] = -np.inf @@ -576,7 +642,8 @@ def detect_peaks_2d( act_sel_pix &= ~np.isnan(act_im_2d) thetaf = np.column_stack( - [amp, py.astype(float), px.astype(float), 0.5 * np.ones(n_peaks)] + [amp, py.astype(float), px.astype(float), + 0.5 * np.ones(n_peaks), 0.5 * np.ones(n_peaks)] ) p_locs = np.column_stack([py.astype(float), px.astype(float)]) @@ -584,7 +651,9 @@ def detect_peaks_2d( sel_vals = act_im_2d[act_sel_pix] - mu_bg lb_f, ub_f = _make_bounds(p_locs, height, width) - thetaf = _lsq_curvefit(thetaf, sel_yx, sel_vals, lb_f, ub_f, max_nfev=5000) + thetaf = _fit_isotropic_then_anisotropic( + thetaf, sel_yx, sel_vals, lb_f, ub_f, max_nfev=5000 + ) p_im = _peak_mask(thetaf, height, width) buffer_mask = _buffer_mask(p_im, buffer_size) @@ -613,9 +682,12 @@ def detect_peaks_2d( ) if thetaf.shape[0] > 0: - s = thetaf[:, 3] + sy = thetaf[:, 3] + sx = thetaf[:, 4] adj_thresh = peak_thresh / ( - np.pi / 2 * s**2 * erf(1 / (np.sqrt(2) * s)) ** 2 + np.pi / 2 * sy * sx + * erf(1 / (np.sqrt(2) * sy)) + * erf(1 / (np.sqrt(2) * sx)) ) thetaf = thetaf[thetaf[:, 0] >= adj_thresh] @@ -634,6 +706,12 @@ def get_act_im_peaks( threshold are computed once across all planes for uniform sensitivity, then each plane is detected independently (:func:`detect_peaks_2d`). + Gaussians have independent, axis-aligned standard deviations, bounded to + ``0.35 <= sigma_y <= 8`` and ``0.35 <= sigma_x <= 1`` pixels. + Each fit starts with an optimized isotropic Gaussian (shared sigma in + [0.35, 1]) before relaxing to independent widths. This favors isotropic + local minima through initialization without an explicit shape penalty. + Parameters ---------- act_im : ndarray of shape (Z, H, W) diff --git a/tests/test_bandsilo_peaks.py b/tests/test_bandsilo_peaks.py index 7721ff8..6a46e8d 100644 --- a/tests/test_bandsilo_peaks.py +++ b/tests/test_bandsilo_peaks.py @@ -8,6 +8,7 @@ """ import unittest +from unittest.mock import patch import numpy as np @@ -22,16 +23,32 @@ def _grid(height, width): ) -def _single_gaussian_plane(height, width, cy, cx, amp, sigma): +def _single_gaussian_plane(height, width, cy, cx, amp, sigma, sigma_x=0.8): """Render one integrated Gaussian over an ``(H, W)`` plane.""" yx = _grid(height, width) - theta = np.array([[amp, cy, cx, sigma]]) + theta = np.array([[amp, cy, cx, sigma, sigma_x]]) return pk.gaussian_peaks_integrated(theta, yx).reshape(height, width) class TestGaussianForward(unittest.TestCase): """gaussian_peaks_integrated analytic properties.""" + def test_anisotropic_total_integral(self): + """Unequal widths integrate to amp*2*pi*sigma_y*sigma_x.""" + theta = np.array([[3.0, 50.3, 12.7, 8.0, 0.7]]) + total = pk.gaussian_peaks_integrated(theta, _grid(101, 27)).sum() + self.assertAlmostEqual(total, 3.0 * 2 * np.pi * 8.0 * 0.7, places=6) + + def test_equal_widths_match_legacy_isotropic(self): + yx = _grid(25, 25) + isotropic = np.array([[2.0, 12.3, 10.7, 0.9]]) + anisotropic = np.column_stack([isotropic, isotropic[:, 3]]) + np.testing.assert_allclose( + pk.gaussian_peaks_integrated(anisotropic, yx), + pk.gaussian_peaks_integrated(isotropic, yx), + rtol=1e-12, + ) + def test_total_integral(self): """Integrated Gaussian summed over a wide grid gives amp*2*pi*s^2.""" yx = _grid(60, 60) @@ -55,6 +72,30 @@ def test_superposition(self): class TestValJac(unittest.TestCase): """_gaussian_peaks_integrated_val_jac value + analytical Jacobian.""" + def test_anisotropic_jacobian_matches_finite_difference(self): + """Both width derivatives are independent and correctly ordered.""" + yx = _grid(35, 19) + theta = np.array([ + [2.0, 16.3, 7.7, 8.0, 0.7], + [1.2, 21.0, 12.0, 3.0, 1.0], + ]) + val, jac = pk._gaussian_peaks_integrated_val_jac(theta, yx) + np.testing.assert_allclose( + val, pk.gaussian_peaks_integrated(theta, yx), rtol=1e-12 + ) + self.assertEqual(jac.shape, (len(yx), theta.size)) + eps = 1e-6 + for k in range(theta.size): + hi = theta.ravel().copy() + lo = theta.ravel().copy() + hi[k] += eps + lo[k] -= eps + fd = ( + pk.gaussian_peaks_integrated(hi.reshape(-1, 5), yx) + - pk.gaussian_peaks_integrated(lo.reshape(-1, 5), yx) + ) / (2 * eps) + np.testing.assert_allclose(jac[:, k], fd, rtol=1e-4, atol=1e-7) + def test_value_matches_forward(self): """The val_jac value equals the forward-only evaluation.""" yx = _grid(20, 20) @@ -87,17 +128,25 @@ class TestLsqCurvefit(unittest.TestCase): def test_recovers_known_gaussian(self): """Fitting noise-free data recovers the generating parameters.""" yx = _grid(21, 21) - true = np.array([[5.0, 10.4, 9.6, 1.3]]) + true = np.array([[5.0, 10.4, 9.6, 6.0, 0.7]]) ydata = pk.gaussian_peaks_integrated(true, yx) - theta0 = np.array([[3.0, 10.0, 10.0, 1.0]]) + theta0 = np.array([[3.0, 10.0, 10.0, 0.5, 0.5]]) lb, ub = pk._make_bounds(theta0[:, 1:3], 21, 21) - # widen sigma/amp bounds for a clean recovery - lb[0::4] = 0.0 - ub[0::4] = np.inf fit = pk._lsq_curvefit(theta0, yx, ydata, lb, ub) - np.testing.assert_allclose(fit[0, 1], 10.4, atol=1e-2) - np.testing.assert_allclose(fit[0, 2], 9.6, atol=1e-2) - np.testing.assert_allclose(fit[0, 3], 1.3, atol=1e-2) + np.testing.assert_allclose(fit, true, atol=1e-2) + + def test_width_limits(self): + """A target wider than either limit cannot exceed the fitted bounds.""" + yx = _grid(41, 21) + true = np.array([[5.0, 20.4, 9.6, 10.0, 1.5]]) + theta0 = np.array([[4.0, 20.0, 10.0, 6.0, 0.8]]) + lb, ub = pk._make_bounds(theta0[:, 1:3], 41, 21) + fit = pk._lsq_curvefit( + theta0, yx, pk.gaussian_peaks_integrated(true, yx), lb, ub + ) + self.assertTrue(np.all(fit.ravel() >= lb)) + self.assertTrue(np.all(fit.ravel() <= ub)) + np.testing.assert_allclose(fit[0, 3:], [8.0, 1.0], atol=1e-2) def test_max_nfev_early_break(self): """max_nfev=1 returns the (clipped) initial parameters immediately.""" @@ -105,10 +154,8 @@ def test_max_nfev_early_break(self): ydata = pk.gaussian_peaks_integrated( np.array([[4.0, 5.0, 5.0, 1.0]]), yx ) - theta0 = np.array([[3.0, 5.0, 5.0, 1.0]]) + theta0 = np.array([[3.0, 5.0, 5.0, 1.0, 1.0]]) lb, ub = pk._make_bounds(theta0[:, 1:3], 11, 11) - lb[0::4] = 0.0 - ub[0::4] = np.inf out = pk._lsq_curvefit(theta0, yx, ydata, lb, ub, max_nfev=1) np.testing.assert_allclose(out, theta0) @@ -132,18 +179,86 @@ def test_non_improving_step_increases_damping(self): self.assertTrue(np.all(out.ravel() <= ub + 1e-9)) +class TestIsotropicWarmStart(unittest.TestCase): + """Two-stage fitting preserves round peaks while permitting elongation.""" + + def test_stages_and_bounds(self): + """Relaxation starts from the fitted shared widths, not the seed.""" + theta0 = np.array([[3.0, 10.0, 10.0, 4.0, 0.8]]) + original = theta0.copy() + lb, ub = pk._make_bounds(theta0[:, 1:3], 21, 21) + iso_fit = np.array([[5.0, 10.3, 9.7, 0.9]]) + final_fit = np.array([[5.0, 10.3, 9.7, 4.0, 0.7]]) + with patch.object( + pk, "_lsq_curvefit", side_effect=[iso_fit, final_fit] + ) as solver: + out = pk._fit_isotropic_then_anisotropic( + theta0, _grid(21, 21), np.zeros(441), lb, ub, + max_nfev=100, + ) + self.assertEqual(solver.call_count, 2) + first, second = solver.call_args_list + self.assertEqual(first.args[0].shape, (1, 4)) + self.assertEqual(first.args[3][3], 0.35) + self.assertEqual(first.args[4][3], 1.0) + np.testing.assert_array_equal( + second.args[0], [[5.0, 10.3, 9.7, 0.9, 0.9]] + ) + np.testing.assert_array_equal(second.args[3], lb) + np.testing.assert_array_equal(second.args[4], ub) + self.assertEqual(first.kwargs["max_nfev"], 100) + self.assertEqual(second.kwargs["max_nfev"], 100) + np.testing.assert_array_equal(out, final_fit) + np.testing.assert_array_equal(theta0, original) + + def test_recovers_round_and_elongated_peaks(self): + """Joint fitting recovers round and tall peaks after relaxation.""" + yx = _grid(65, 31) + true = np.array([ + [5.0, 15.3, 8.7, 0.8, 0.8], + [8.0, 40.4, 22.2, 7.0, 0.7], + ]) + theta0 = np.array([ + [3.0, 15.0, 9.0, 0.5, 0.5], + [4.0, 40.0, 22.0, 0.5, 0.5], + ]) + lb, ub = pk._make_bounds(theta0[:, 1:3], 65, 31) + out = pk._fit_isotropic_then_anisotropic( + theta0, yx, pk.gaussian_peaks_integrated(true, yx), lb, ub + ) + np.testing.assert_allclose(out, true, atol=0.01) + + def test_exact_isotropic_minimum_stays_isotropic(self): + """Relaxing an already exact round fit does not change it.""" + theta = np.array([[5.0, 10.3, 9.7, 0.8, 0.8]]) + yx = _grid(21, 21) + lb, ub = pk._make_bounds(theta[:, 1:3], 21, 21) + out = pk._fit_isotropic_then_anisotropic( + theta, yx, pk.gaussian_peaks_integrated(theta, yx), lb, ub + ) + np.testing.assert_array_equal(out, theta) + + def test_empty_parameters(self): + """No peaks need no optimization.""" + theta = np.empty((0, 5)) + out = pk._fit_isotropic_then_anisotropic( + theta, _grid(3, 3), np.zeros(9), np.array([]), np.array([]) + ) + self.assertEqual(out.shape, (0, 5)) + + class TestSmallHelpers(unittest.TestCase): """_make_bounds / _peak_mask / _buffer_mask.""" def test_make_bounds(self): - """Means bound to +/-1.5 px (clipped); sigma to [0.35, 5].""" + """Means clip to the image; sigma_y <= 8 and sigma_x <= 1.""" plocs = np.array([[0.0, 9.0]]) lb, ub = pk._make_bounds(plocs, 10, 10) # y lower clipped to 0; x within range self.assertAlmostEqual(lb[1], 0.0) self.assertAlmostEqual(ub[2], 9.0) # min(W-1, 9+1.5) = 9 - self.assertAlmostEqual(lb[3], 0.35) - self.assertAlmostEqual(ub[3], 5.0) + np.testing.assert_array_equal(lb[3:], [0.35, 0.35]) + np.testing.assert_array_equal(ub[3:], [8.0, 1.0]) def test_peak_mask_and_empty(self): """Peak centers mark the mask; an empty parameter set marks nothing.""" @@ -174,7 +289,7 @@ def test_rejects_integer_peak_with_others(self): # a blob centered exactly on the integer seed -> fit stays near-integer act = _single_gaussian_plane(h, w, 5.0, 5.0, 8.0, 1.1) labeled = np.ones((h, w), dtype=int) - thetaf = np.array([[1.0, 10.0, 10.0, 1.0]]) # existing peak in cc + thetaf = np.array([[1.0, 10.0, 10.0, 1.0, 0.8]]) # existing peak in cc p_locs = np.array([[10.0, 10.0]]) reject_mask = np.zeros((h, w), dtype=bool) new_thetaf, _, _, _ = pk._process_cc_new_peak( @@ -189,7 +304,7 @@ def test_rejects_flat_peak_alone(self): act = np.zeros((h, w)) labeled = np.zeros((h, w), dtype=int) labeled[3:12, 3:12] = 1 - thetaf = np.zeros((0, 4)) + thetaf = np.zeros((0, 5)) p_locs = np.zeros((0, 2)) reject_mask = np.zeros((h, w), dtype=bool) new_thetaf, _, _, _ = pk._process_cc_new_peak( @@ -203,7 +318,7 @@ def test_accepts_moving_peak(self): h = w = 15 act = _single_gaussian_plane(h, w, 5.4, 5.6, 6.0, 1.1) labeled = np.ones((h, w), dtype=int) - thetaf = np.zeros((0, 4)) + thetaf = np.zeros((0, 5)) p_locs = np.zeros((0, 2)) reject_mask = np.zeros((h, w), dtype=bool) new_thetaf, _, _, _ = pk._process_cc_new_peak( @@ -223,7 +338,7 @@ class TestRefineResidualPeaks(unittest.TestCase): def test_empty_support_returns_unchanged(self): """No support (labels max 0) exits immediately, returning thetaf.""" h = w = 10 - thetaf = np.array([[1.0, 5.0, 5.0, 1.0]]) + thetaf = np.array([[1.0, 5.0, 5.0, 1.0, 0.8]]) p_locs = np.array([[5.0, 5.0]]) act = np.zeros((h, w)) empty2d = np.zeros((h, w), dtype=bool) @@ -262,7 +377,19 @@ def test_no_peaks_returns_empty(self): peak_thresh=100.0, # unreachable peak_th=100.0, ) - self.assertEqual(out.shape, (0, 4)) + self.assertEqual(out.shape, (0, 5)) + + def test_elongated_blob_detected(self): + """A tall, narrow blob is fit as one anisotropic Gaussian.""" + act = _single_gaussian_plane(65, 25, 32.3, 12.4, 10.0, 7.0, 0.7) + out = pk.detect_peaks_2d( + act, np.zeros_like(act, dtype=bool), + mu_bg=0.0, sigma_bg=0.05, peak_thresh=1.0, peak_th=3.0, + ) + self.assertEqual(out.shape, (1, 5)) + np.testing.assert_allclose( + out[0], [10.0, 32.3, 12.4, 7.0, 0.7], atol=0.05 + ) def test_single_blob_detected(self): """A single blob yields one peak near its true center.""" @@ -286,7 +413,7 @@ def test_close_pair_adds_residual_peak(self): recovered by the residual-peak refinement loop.""" act = _single_gaussian_plane( 34, 34, 16.4, 15.0, 10.0, 1.2 - ) + _single_gaussian_plane(34, 34, 16.4, 17.5, 9.0, 1.2) + ) + _single_gaussian_plane(34, 34, 16.4, 16.8, 9.0, 1.2) out = pk.detect_peaks_2d( act, np.zeros((34, 34), dtype=bool), @@ -302,6 +429,15 @@ def test_close_pair_adds_residual_peak(self): class TestGetActImPeaks(unittest.TestCase): """get_act_im_peaks 3-D driver and its guard branches.""" + def test_elongated_blob_seed(self): + """The 3-D driver retains its [z, y, x] output for unequal widths.""" + rng = np.random.default_rng(42) + act = rng.normal(0.0, 0.01, (1, 65, 25)) + act[0] += _single_gaussian_plane(65, 25, 32.3, 12.4, 10.0, 7.0, 0.7) + seeds = pk.get_act_im_peaks(act, peak_th=6.0, buffer_size=3) + self.assertEqual(seeds.shape, (1, 3)) + np.testing.assert_allclose(seeds[0], [0.0, 32.3, 12.4], atol=0.1) + def _act_im(self, seed=0): """Two-plane activity image with a blob per plane, low noise, NaNs.""" rng = np.random.default_rng(seed) From a0c908a8e07a1e50bcc769b5525dbda177c64d5b Mon Sep 17 00:00:00 2001 From: m-xie Date: Fri, 11 Sep 2026 00:26:29 +0000 Subject: [PATCH 32/55] change default peakth --- src/giant_python/models/params.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/giant_python/models/params.py b/src/giant_python/models/params.py index 0eb3861..99f07db 100644 --- a/src/giant_python/models/params.py +++ b/src/giant_python/models/params.py @@ -149,7 +149,7 @@ class SiloParams: phi: Optional[float] = None tau_s: Optional[float] = None photon_scale: Optional[float] = None - peakth: float = 8.0 + peakth: float = 7.0 activity_channel: int = 0 draw_user_rois: bool = False interactive: Optional[bool] = None From 846c77241a51b94553bef1138c2639526e5e00a7 Mon Sep 17 00:00:00 2001 From: m-xie Date: Fri, 11 Sep 2026 00:31:38 +0000 Subject: [PATCH 33/55] print number of detected sources --- src/giant_python/bandsilo/pipeline.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/giant_python/bandsilo/pipeline.py b/src/giant_python/bandsilo/pipeline.py index a6a23e6..e9262af 100644 --- a/src/giant_python/bandsilo/pipeline.py +++ b/src/giant_python/bandsilo/pipeline.py @@ -1010,6 +1010,7 @@ def _localize( act_im, peak_th=peak_th, buffer_size=params.peak_buffer ) act_im_peaks = source_seeds.copy() + print(f"Number of detected sources: {act_im_peaks.shape[0]}") result = fit_sources( source_seeds, From 3259392f599fbd6eb22890a6df7b2c47ef87aa84 Mon Sep 17 00:00:00 2001 From: m-xie Date: Fri, 11 Sep 2026 00:39:02 +0000 Subject: [PATCH 34/55] restrict size of final spatial profiles --- src/giant_python/bandsilo/nmf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/giant_python/bandsilo/nmf.py b/src/giant_python/bandsilo/nmf.py index d671a0f..dccc9c6 100644 --- a/src/giant_python/bandsilo/nmf.py +++ b/src/giant_python/bandsilo/nmf.py @@ -499,8 +499,8 @@ def fit_gaussian_profiles( with torch.no_grad(): optim_loc.clamp_(min=loc_min, max=loc_max) - optim_scale[:, 0].clamp_(min=0.3, max=5) - optim_scale[:, 1].clamp_(min=0.3, max=5) + optim_scale[:, 0].clamp_(min=0.3, max=2) + optim_scale[:, 1].clamp_(min=0.3, max=2) if epoch % n_motions == n_motions - 1: total_loss = 0 From 3a3d54a1e54c1db8c690c7eb32583d6684af57da Mon Sep 17 00:00:00 2001 From: Michael Xie Date: Mon, 21 Sep 2026 01:16:13 -0700 Subject: [PATCH 35/55] refactor package to reorganize and rename files --- .coverage-workflows | Bin 0 -> 77824 bytes .flake8 | 2 + README.md | 134 +- docs/source/architecture.rst | 200 +++ docs/source/index.rst | 192 ++- docs/source/migration.rst | 301 ++++ examples/extract_band.py | 320 ++++ pyproject.toml | 4 +- src/giant_python/__init__.py | 107 +- src/giant_python/bandsilo/__init__.py | 13 - src/giant_python/bandsilo/annotate.py | 340 ----- src/giant_python/bandsilo/background.py | 1313 ----------------- src/giant_python/bandsilo/geometry.py | 364 ----- src/giant_python/bandsilo/gui.py | 694 --------- src/giant_python/bandsilo/hdf5.py | 450 ------ src/giant_python/bandsilo/pipeline.py | 1034 ------------- src/giant_python/cli.py | 137 +- src/giant_python/extraction/__init__.py | 1 + src/giant_python/extraction/band/__init__.py | 1 + src/giant_python/extraction/band/activity.py | 333 +++++ .../extraction/band/annotation.py | 216 +++ .../extraction/band/background.py | 505 +++++++ src/giant_python/extraction/band/geometry.py | 234 +++ src/giant_python/extraction/band/inputs.py | 337 +++++ .../band/localization.py} | 30 +- .../extraction/band/motion_binning.py | 110 ++ src/giant_python/extraction/band/operators.py | 321 ++++ .../extraction/band/result_assembly.py | 180 +++ src/giant_python/extraction/band/rois.py | 158 ++ .../band}/summary_images.py | 167 ++- .../{bandsilo => extraction/band}/traces.py | 59 +- .../band}/trial_data.py | 127 +- src/giant_python/extraction/band/types.py | 34 + src/giant_python/extraction/band/workflow.py | 591 ++++++++ src/giant_python/gui/__init__.py | 10 +- src/giant_python/gui/draw_rois.py | 52 - src/giant_python/gui/parameters_gui.py | 118 ++ src/giant_python/gui/roi_editor.py | 198 +++ src/giant_python/io/__init__.py | 54 +- src/giant_python/io/annotations.py | 91 ++ src/giant_python/io/experiment_summary.py | 257 ++++ src/giant_python/io/hdf5.py | 88 +- src/giant_python/io/slap2.py | 150 +- src/giant_python/io/tiff.py | 60 +- src/giant_python/io/trial_table.py | 48 + src/giant_python/math/__init__.py | 28 - src/giant_python/math/baseline.py | 35 - src/giant_python/math/deconv.py | 31 - src/giant_python/math/interpolation.py | 40 - src/giant_python/math/nmf.py | 41 - src/giant_python/math/peaks.py | 95 -- src/giant_python/math/registration.py | 124 -- src/giant_python/math/variance.py | 29 - src/giant_python/models/__init__.py | 33 +- src/giant_python/models/alignment.py | 107 -- .../{experiment.py => experiment_summary.py} | 128 +- src/giant_python/models/params.py | 333 +++-- src/giant_python/models/trial_table.py | 87 +- src/giant_python/numerics/__init__.py | 4 + .../{bandsilo => numerics}/baseline.py | 2 +- src/giant_python/numerics/interpolation.py | 30 + src/giant_python/numerics/morphology.py | 60 + .../{bandsilo => numerics}/peaks.py | 160 +- src/giant_python/parallel.py | 9 +- src/giant_python/pipeline/__init__.py | 51 +- src/giant_python/pipeline/annotate.py | 55 + src/giant_python/pipeline/base.py | 23 - src/giant_python/pipeline/extract.py | 221 ++- src/giant_python/pipeline/organize.py | 142 -- src/giant_python/pipeline/pipeline.py | 251 ++-- src/giant_python/pipeline/register.py | 173 --- src/giant_python/{bandsilo => }/progress.py | 4 +- tests/test_acquisition_boundaries.py | 215 +++ tests/test_band_workflow.py | 460 ++++++ tests/test_bandsilo_annotate.py | 82 +- tests/test_bandsilo_background.py | 73 +- tests/test_bandsilo_baseline.py | 4 +- tests/test_bandsilo_geometry.py | 25 +- tests/test_bandsilo_gui.py | 12 +- tests/test_bandsilo_hdf5.py | 38 +- tests/test_bandsilo_nmf.py | 10 +- tests/test_bandsilo_peaks.py | 196 ++- tests/test_bandsilo_phase_filter.py | 198 ++- tests/test_bandsilo_pipeline.py | 470 +++++- tests/test_bandsilo_progress.py | 4 +- tests/test_bandsilo_summary_images.py | 110 +- tests/test_bandsilo_traces.py | 19 +- tests/test_bandsilo_trial_data.py | 25 +- tests/test_cli.py | 232 ++- tests/test_configuration.py | 184 +++ tests/test_gui.py | 32 - tests/test_io.py | 62 +- tests/test_io_architecture.py | 281 ++++ tests/test_io_edges.py | 523 +++++++ tests/test_math.py | 110 -- tests/test_models.py | 95 +- tests/test_motion_convention.py | 89 ++ tests/test_no_placeholders.py | 335 +++++ tests/test_numeric_architecture.py | 296 ++++ tests/test_numeric_edges.py | 184 +++ tests/test_package_metadata.py | 103 +- tests/test_parallel.py | 7 + tests/test_parameter_gui.py | 105 ++ tests/test_path_summary.py | 374 +++++ tests/test_pipeline.py | 260 +--- tests/test_staging_example.py | 547 +++++++ tests/test_workflows.py | 320 ++++ 107 files changed, 11442 insertions(+), 7069 deletions(-) create mode 100644 .coverage-workflows create mode 100644 docs/source/architecture.rst create mode 100644 docs/source/migration.rst create mode 100644 examples/extract_band.py delete mode 100644 src/giant_python/bandsilo/__init__.py delete mode 100644 src/giant_python/bandsilo/annotate.py delete mode 100644 src/giant_python/bandsilo/background.py delete mode 100644 src/giant_python/bandsilo/geometry.py delete mode 100644 src/giant_python/bandsilo/gui.py delete mode 100644 src/giant_python/bandsilo/hdf5.py delete mode 100644 src/giant_python/bandsilo/pipeline.py create mode 100644 src/giant_python/extraction/__init__.py create mode 100644 src/giant_python/extraction/band/__init__.py create mode 100644 src/giant_python/extraction/band/activity.py create mode 100644 src/giant_python/extraction/band/annotation.py create mode 100644 src/giant_python/extraction/band/background.py create mode 100644 src/giant_python/extraction/band/geometry.py create mode 100644 src/giant_python/extraction/band/inputs.py rename src/giant_python/{bandsilo/nmf.py => extraction/band/localization.py} (96%) create mode 100644 src/giant_python/extraction/band/motion_binning.py create mode 100644 src/giant_python/extraction/band/operators.py create mode 100644 src/giant_python/extraction/band/result_assembly.py create mode 100644 src/giant_python/extraction/band/rois.py rename src/giant_python/{bandsilo => extraction/band}/summary_images.py (84%) rename src/giant_python/{bandsilo => extraction/band}/traces.py (93%) rename src/giant_python/{bandsilo => extraction/band}/trial_data.py (76%) create mode 100644 src/giant_python/extraction/band/types.py create mode 100644 src/giant_python/extraction/band/workflow.py delete mode 100644 src/giant_python/gui/draw_rois.py create mode 100644 src/giant_python/gui/parameters_gui.py create mode 100644 src/giant_python/gui/roi_editor.py create mode 100644 src/giant_python/io/annotations.py create mode 100644 src/giant_python/io/experiment_summary.py create mode 100644 src/giant_python/io/trial_table.py delete mode 100644 src/giant_python/math/__init__.py delete mode 100644 src/giant_python/math/baseline.py delete mode 100644 src/giant_python/math/deconv.py delete mode 100644 src/giant_python/math/interpolation.py delete mode 100644 src/giant_python/math/nmf.py delete mode 100644 src/giant_python/math/peaks.py delete mode 100644 src/giant_python/math/registration.py delete mode 100644 src/giant_python/math/variance.py delete mode 100644 src/giant_python/models/alignment.py rename src/giant_python/models/{experiment.py => experiment_summary.py} (56%) create mode 100644 src/giant_python/numerics/__init__.py rename src/giant_python/{bandsilo => numerics}/baseline.py (99%) create mode 100644 src/giant_python/numerics/interpolation.py create mode 100644 src/giant_python/numerics/morphology.py rename src/giant_python/{bandsilo => numerics}/peaks.py (84%) create mode 100644 src/giant_python/pipeline/annotate.py delete mode 100644 src/giant_python/pipeline/base.py delete mode 100644 src/giant_python/pipeline/organize.py delete mode 100644 src/giant_python/pipeline/register.py rename src/giant_python/{bandsilo => }/progress.py (92%) create mode 100644 tests/test_acquisition_boundaries.py create mode 100644 tests/test_band_workflow.py create mode 100644 tests/test_configuration.py delete mode 100644 tests/test_gui.py create mode 100644 tests/test_io_architecture.py create mode 100644 tests/test_io_edges.py delete mode 100644 tests/test_math.py create mode 100644 tests/test_motion_convention.py create mode 100644 tests/test_no_placeholders.py create mode 100644 tests/test_numeric_architecture.py create mode 100644 tests/test_numeric_edges.py create mode 100644 tests/test_parameter_gui.py create mode 100644 tests/test_path_summary.py create mode 100644 tests/test_staging_example.py create mode 100644 tests/test_workflows.py diff --git a/.coverage-workflows b/.coverage-workflows new file mode 100644 index 0000000000000000000000000000000000000000..4dd485a687de3008d124fab184031a0d260e589a GIT binary patch literal 77824 zcmeHQ33MCBdEQ+t?qhj~A}NX_MDY;CONVva5^0M%OiC0fK?em1T!Kpw5`cwqNTM8C zgyJ@F^IE4#-8QH4;+MuwWGl~(tvI$UIc<|ZCytY*wvzae@|@VQ9b0i@iL?%)|LiV+ z1wgq1cH7u_vya3q_W1UnZ~poJ`Db=8TYI~`5zao$hx}~BzL+Q=NRnu`+X;d&!+$aS zCpJCo(1|aQlCu5l?V5?!`+j0&wi6cJPJ-EMoyD|RJeK#&U$!haI?P^uv#}LU;0-f? z8Ndu+2L56gi1k|ZrRC-1(Xj|S=;I>c(Hf6yM}LiS;=kF$FNLtZx<@diBh$OxCXIvVDNkk5i+;I+_&t-}3i&*dG0yCR$i zAc7&UpAC)L_j9Ao!UfXPBHW>fcmO~+uP1=6u+NijZ?uQFVJ^f4++0|cVxD)X(bV|K zBPM-maWVNRPRJ7*a>JiFEjMr=m6eUd0OaOFLrD6;5F2ohaN%Y ze_MpN`@8`mn}M)5;^hN&?hxmWM!2DAGuVtsOKd=%JU4(ffHE{7-YGpx#!l+yY3`IR zmu?;g;R1Yu*3^yY(QvaMb0F#;1j_eC{XtYXaEJKd2!FjhuXk^Rmoj_9Wiw4#rt+Is zoC(TNiHdhc1KvBLT(a<+?WsGPg=_@)W;@CRXh5A=SGuB@BtR>azL&Aa#_CX&%n67Jr#)LVjAYcm;mkmb) zZd6smC=YJ_02g9CT&r|eGz))jMFQ1XXw;QjP^NWW6D%vS5Qn4`4?Z4zX}CJIF^Pg8rhf zf_fs~laf~iNPSOm4_7q`mqYd5!iU3NaI45|g;yqy+8ajLFtS472Kyk-`#3fr`UbdT zV;G+44ci47vc1xp6h?RyVR#NS2r?y{3o`TDpwcDk7wq``Y+y+8SI7;Es9^h1$go}9noE=>v z2qP%gK{jl6hd4MX`BFhJU~D8)DiLM_oJrVFF*l73nh(<*RK%`@=o3ZB2Go>`Rg?PC zrY15Lm1?kPoLpgO3HaGWUZt;8nr3C3Oet9pSZQb?$(0KcF5?h=6`z1CqFC&MB6W$Y zqz00y(F#(M%oPIu@P-+{3}6N@1DFBK0A>I)fEmCHUDp z1oJL@;0-f?8Ndu+1~3Dd0n7kq05gCYzzkppFawx@k0k>ZN>j-sUJF@JKx!(=rFQ_9 zwJu!Ry3hoyX z*`yFRu+!U06S0QdRyrMFm@?w@d)*@}=W9LW<(zAIcht`XB4OwHjU9n)En7w-BYeOa z4!NBkFB^!sg5m*B)ax7yu?Jlt-W!H_Bx_p$(wYlogbTw{(5K7K4{<((($x$o6?vk- zG^k;(CmMo@tb72#);9rIah_lY`OyAhAAb-@)juDAR^$SdMHfs!g~_={Fy|ml=y3{D zWD)j2BVc!`itXdwtj~KmLBq~@@Ps9*K0zw;y$yiep(-*P2=I}FmoPypI662N@HeVTOE?Pi%tE6sn8@uB_}Nnf z&sd}CGkBP8&PMor-o>G$(ewcl%3o&G`Az26h!qk$nPUS~Dn-=Zot9*&0@ZJ|+w zUZpxZvO5R75rp1d3FwuIWDkaTPlyYnQtY0s9E}aJJ|Fm=;An3Jpeb`oUf#(a0yTqq ziGc_}_RRvwMn%~LNU`z>a|;2ps~j+Ma#2~%0AYHEhY^EiiWvm3u#XKcLfI`ttpT)<^bQ`A}udwsks=nW>MU2g@zoEnx{Ccy3U zLbf|B0Hn`TG7-XI1_)&ZlE|5)^jruebC}eFP2eL~qk6D>$ioJ_hmrbhGXQK$E<9!x zNJ2<(AVNzW39DC6Sjng&VS9A|tIW?Og+)~eAh&A)a-pL9iy>Hsuj{21iz?v;!O#1V z)pKYRlPW=`dHQV>z$()K2^K;pPe5uvtRn%nFc)5BmLmv(c%chE3R)@nq!_eM6fq|l zm-eTc&r`FtMf9WQ8tN{(hdiUXpjmD-5`S+P(bpTEGR)FHryph9jK%hZjWzw)#G7Us ze_-_JerY{vvsqcI$?~YhsrwuA5lcBapZG2F1bGmF-|*H)#~R37@|9IzGUNZ&PC91I z2~tLtNIhsrOE(>>$QLCu{%_tu$BOd>OSCJH&Nl6)V=Hn3%AzaNJ0;`)`79mlR2MrV z{%_nx$Cjx31S$TX=b~dB>LRBapTK)VKOI}9E`B2ZuUEfFlktC@`n06Q|Fv#9wozSL zvf}@_d+FF3b)O-R|7+BzKNT77g`{9m<+j#Vm?T^9dW zs(_Y?|7R#s3iYGF^16JSPUGtK$&Vf|7Ovo=sJ^hkyU?hI=OP zg_r@%0A>I)fEmCHURzv^kaTXWj8?QYFv>DtmdIKQCo=)=X6Z+fVzr>;!J3$ISpPu{tp z7ETP-QW3~z?Ol5=w~Sz{!K_JXQvxNUVc z^-kSnPuzbmG4+d2-$|c;-FRl|OXqveAET?Q;Cy@4(ZfRC(}ah0)MB3dE2(+p zS~y!~nGIiO&89Ye5}qBbtAKCa71ZIbS=Zu6-hcbveky*!u#@fo$EkR{hZs0}A^fHD zHKcVG!kD$MuKg?IwKb-qa`;+bezePuG^GG7*KQ~)gJX-zsO=Onax}hPP?~X+J+iD6 zohg0mU;dVujK9+Ji8jO4YxTs$ojC~^?y33r>cqZ zMI~^0xP*G>jri#0!Xxj;*E7NRD^E_GES`!_?Rn$Kg#X;?Yb&OzpSf`DE_(HmmnPml zd9G*%+*3c}Xc2leln9h>`QBnw?l%|1ezZ2hbQ)83bhWUFN-vyjC&-j`j7#T*B%%4@ zF-l(q&nzgKPZ|qhXGYj+FNmX0nhIT9UwZD4z&LgKMHWM#BzUOlF*aHvV zv-hno;>@|?G7DU=$nxv&>^n8}>n|=L?t0O6TDNBG^4W$8Gn`p&rbdd1amZOMK`bJc z5QKL8_;JgeQWKn9U^=?=nD)#BPfVVE;jzb`zFhJrO{o!`Fj5b}Eq2Hflp9e#ZX()C z3~-{^uFVgMPEhyC@6zy9f;>?f1Yx@xuOLi@T9P&zyT7j|&h+np=&|8F`%gt*dFbLX$BENZ7caZOUY|XE zaX*E||J!U&5b)c7?=$Z(uQTVEUot;uo@M@(d5U?Ad6@Yo^EKuR%;%WTFn2R!%mIdH zMi`dqXSOokOeeFNS;j128kriVf|E|mwle^aH_QNL z05gCYzzkppFawwY%m8KpGk_Vu4CKcEioeOWHX&(k6_S<~A!%+FlBOmhnLl4h8XJXV z-aH{`Xb_V6dLgN+6O!6mA(=Z@NNQ??WX>ERvD<~Dx>`u8s)VGnQb=ac7LtkzA(=Hx zNXpBFq^wLxN=t=g=1d_eDG`zxGlZnLSV)SBgru-gND2ysgkgllW)l*tRY)urAu*eU z#AFf@qftlec*YQJ7M*UnM5_YcN z*b&&)vSl-X<|j* zTa~ZIEaW`kABIeKsIVv}%5>%uyzMzJXFGkahd4JMH~@mKQK`UX>-nx$zFzXjYT<2F zR|{2M9Kc;_byO%}b=E=8SQuP+5yNt;&Jgc)aYNn+9|Aes)WH-Mo8=;02rXcKi*h=V zoI@e@pbLaWkgXr}Wy~|z!j6mvIV5VB~$SAvVGSey#GA$FuQSw;=m` zHyzukuI%Bc-_M3dU0y%y5uP!37ad!o?lUBnnfHIX!1-P}wnkmC`5+f!p;98PsZqaF z{d~mB2LO8xOUF9Z72U_XS)cbXD?Oo6ed)b{U^If5sZ-w&csSnAMM9&1Uafv@9c116 zJt00C7y^8|`uHgq0r(B-v%|V0-UHqUVy9{Y9V=F*Ic^_X?%TSPjxAOOG{^>q!d@Tm z6qiqztl`bu=vb3#z>*nUrQDDtrOmQ`E0wp?li3F{4VK5qa;70Z>k z7E;o3?0zKbvR*n?r&5`TVZc)5?T@6CBSXVWK*%L33Mnpmf0K$OB(q?PRfLsA7mHM^ zv1t}@p^EZKrVuIW0u$*Y=8`pp=8u~DH9gc->f6*-N=rUwylDKou^WEnkFY&p>(#Yb zFIvB>eaX7P@{aay%Q4Fa^F{M9^9Iv!VLs00RGz*X?3M?h$WzKw zUu2lvA=V%CA(-tO=~zj=FsORC5=~r(?kJI4Gf^f`qN2Hw9mkyuVW;2g9$`6O>me`a zT+6#}yz>WlF65-mSv@om!H0cpa1qpvvYh*SrqgJdRqj#+P-c&B&)LOICt;c;-l&R< zG8?=>xoMY7U|8O>xn8*|PQr@qg>}kXjY-60E$n2zMLnCYk?ZMLVfvB(rJ~Eq40W*h zd|(%XqkD5UlW92a{SbP)AmU-AM)Mve1uuq`F4V6Hz0_iax%)OcX3RIEs5=)j*}s{N zRp*Q02T_x`37Pq)x^B%Uq*g1(ixUgxURcY?-nOn)F{8<9#!74q&osX%ady=2xsdmP}&$ z`A~3#_wgRoKUJi>-!TcCIRsd!ToIDkvUXg73L}=HZ58{$BJ7|L-D#Cq1;2`1%msk|Q zR97>gROE>Q&Awr;Cko>hV#{xR6Mz-x2_|ph^v?&N6}bRq(RJNnt${|s?o<^!V*q65 zJb1zqRi7XYYVBj_UL$hZOhPRi{6#Yr&;s0oYZFU^81-F!mhv32$li*Z_QP zF5t4JX>v#?5sZVPt+xVTP7TW}6Cl2N&|v`}eV&qu5DqgyC@YXe&g4Bf4wHJY34CDA zVN?%R)`PPR)-Q~;Ip0BZt2wltTI2B6jtgO+O7r2g^KboMZ@WV zdoQh6)Xc!$p;1igbb-H+i zeX3!%c@N;s$y=tZA)-Df;N{e+la-3(*)oS@dUgY5zCL2QVW(UB0WjZD5oy?I(=NcM z&Xc(8VW;+;0H@6x4)hqNH01BE#9^n-?Et~#DMadpkqr)jDajKi?S+xvZGe=MIhJWt zvUQLapl{j=kolII3{raK&(Q~PIl1`hB+Lr&dwKyjUzJZvDGLX;ZUMk11!aO>p>zdD zV6ekA?DfI>`=D~&Jph_h#5!H(6K(F>Hv?L}<}sOPaiG3y6JX^O2u>%N6dB##t%hJ? z73=8&EX4&Y>yzlYT@654bH8^Z04lC?X}vN%8`O|d)=YD(2f&v@99X zC{x;Ot{pc6rm{|xgc(nNS%2#)09ICuWWbW=-n|k)m33<*L9_g~V+DXKGf+uznHP5~ zR}nV#c2D0jfK{dpDQTsb^|veqOl7t-i78gs-X(yjtQI5@CC|R)CiR3A{kvl^AgWaG zGB58~q#mry3hY=2uqxHqG>`9Cpq{)kzu(seu*wt_S*^5rt!Vy#x#6D)=26DOl-vHz z7Pi$`e{B7PwbAmrdLQ}#dW>=7N#lNFt>HIt3*ImTm;uZHW&ksQ z8Ndvv%0SLjsAWA^iGg=v3V@R72%#wYA%FpuW)Z0Q+U7@ficHS0LCcEmI^!Qs)u-Lu z;B=1EJPnzU;!qrDMGKz1gMMI3MvWK(?oeskZO*@2B$Fibb=zhoMnHU-!IJZf;xc&> zfwwSuH~Et^+muY*l0iy6I$Lvo?|eE5v-)!SlvKZ@l(Jr&UUdLd{WrFprzK98@r=Qp zEjiCTnHE&)4uWxy{9oBmk|=frZB|kz>% literal 0 HcmV?d00001 diff --git a/.flake8 b/.flake8 index abe8e86..765c418 100644 --- a/.flake8 +++ b/.flake8 @@ -7,3 +7,5 @@ exclude = venv, .conda max-complexity = 10 +# Black's PEP 8-compliant slice formatting conflicts with pycodestyle E203. +extend-ignore = E203 diff --git a/README.md b/README.md index b09dc57..010bc8d 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,28 @@ [![License](https://img.shields.io/badge/license-MIT-brightgreen)](LICENSE) ![Code Style](https://img.shields.io/badge/code%20style-black-black) [![semantic-release: angular](https://img.shields.io/badge/semantic--release-angular-e10079?logo=semantic-release)](https://github.com/semantic-release/semantic-release) -![Interrogate](https://img.shields.io/badge/interrogate-97.5%25-brightgreen) -![Coverage](https://img.shields.io/badge/coverage-98%25-brightgreen) ![Python](https://img.shields.io/badge/python->=3.10-blue?logo=python) -Python translation of the [GIAnT-MATLAB](https://github.com/AllenNeuralDynamics/GIAnT-MATLAB) analysis package. +An incremental Python port of [GIAnT-MATLAB](https://github.com/AllenNeuralDynamics/GIAnT-MATLAB), currently focused on **SLAP2 BandSILo extraction and annotation from a pre-registered trial table**. This is not a full MATLAB pipeline replacement. + +## Implemented scope + +| Capability | Status | +| --- | --- | +| BandSILo extraction and ROI annotation | Implemented for pre-registered SLAP2 band recordings | +| Trial-table loading | Implemented; accepts an exact filename or a loaded `TrialTable` | +| Generic HDF5 decoding and parameter-group writing | Implemented shared helpers; not a promise of lossless support for every reference field | +| `ExperimentSummary.from_h5()` / `.to_h5()` | Implemented for the emitted band-summary subset, with eager arrays | +| Raw acquisition and alignment reading | Implemented SLAP2 reader access, saved alignment decoding and native TIFF reading | +| Trial-table writing and `AlignmentData` model codecs | Not available; placeholder APIs removed | +| Organization, registration, standard SILo, deconvolution | Not implemented; no placeholder stages or functions are retained | +| Config-file construction, checkpoint/resume | Not implemented; no `from_config()` or `run_all()` API | + +The pre-registration inputs must already include the raw recordings and reference TIFFs, alignment files, and the band registration lookup table. Python consumes these externally generated assets; it does not perform registration. See [architecture](docs/source/architecture.rst) and [API and data conventions](docs/source/migration.rst). + +### Placeholder removal — 2026-09-20 + +The earlier requirement to retain future placeholders is superseded: all unimplemented placeholder functions, files and their exports have been removed. This includes the generic numerical stubs, future pipeline stages, `AlignParams`, `AlignmentData`, unsupported IO wrappers and model writers. Working band science, peak fitting, the band-specific NMF solver, native readers, HDF5 writing/codecs and the `Slap2Reader` typing protocol remain. The removal does not alter scientific algorithms. The MATLAB file schemas below remain useful external-format references, not promises of supported Python APIs. ## Level of Support - [ ] Supported: We are releasing this code to the public as a tool we expect others to use. Issues are welcomed, and we expect to address them promptly; pull requests will be vetted by our staff before inclusion. @@ -31,36 +48,105 @@ Alternatively, if using `uv`, run uv sync ``` +Verbose progress uses the declared `tqdm` dependency, imported only when needed. The existing ROI editor uses OpenCV and tkinter (the latter must be available in the Python installation), not napari. Acquisition and GUI imports remain lazy; ordinary model access does not open a GUI. + +## Supported usage + +Supply the **actual** trial-table filename; it need not have a conventional basename. The following function can also receive an already loaded `TrialTable`: + +```python +from giant_python import ( + AnnotationOptions, BandSiloParams, ExecutionOptions, + extract_band_sources, +) + + +def extract_registered_table(table_or_filename): + return extract_band_sources( + table_or_filename, + BandSiloParams(), + execution=ExecutionOptions(max_workers=6, verbose=True), + annotations=AnnotationOptions(enabled=False, interactive=False), + ) +``` + +The returned `ExperimentSummary` is also written beneath the loaded table's `savedr`, in the `source_extraction` subdirectory. Moving or copying the table alone does **not** change `savedr` or `datadr`. For staging and explicit in-memory directory overrides, use [examples/extract_band.py](examples/extract_band.py). + +For a chainable session, use the same services through `Pipeline`: + +```python +from giant_python import ( + AnnotationOptions, BandSiloParams, ExecutionOptions, Pipeline, +) + + +def extract_session(table_or_filename): + pipeline = Pipeline.from_trial_table( + table_or_filename, + params=BandSiloParams(), + execution=ExecutionOptions(max_workers=6), + annotations=AnnotationOptions(enabled=False, interactive=False), + ).extract() + return pipeline.summary +``` + +`Pipeline` retains the exact source filename when known and accepts a loaded model without serializing/reloading it or guessing a source path from `savedr`. Use `summary.paths` and each `PathSummary.path_index` (zero-based); disk groups are `Path{path_index + 1}`. Arrays are eager, and per-source/ROI entries can share array storage rather than owning independent copies. + +The public workflow methods are `from_trial_table()`, `annotate()` and `extract()`; `extract_sources(...)` is the standalone generic service, not a second `Pipeline` method. `from_config()`, `organize()`, `register()` and `run_all()` are absent, as are the constructor options/attributes `save_dir` and `align_params`. Use the loaded table's `datadr` and `savedr` for explicit directory overrides. Generic `save_struct_h5()` was removed; use the working summary codec for supported band outputs rather than treating the parameter-group writer as a general struct serializer. + +To use existing ROIs without drawing, pass `AnnotationOptions(enabled=True, interactive=False)`. Missing required annotations fail rather than silently omitting ROIs. The explicit `annotate_rois(...)` service or `Pipeline.annotate()` loads existing annotations or draws them when the interactivity policy allows it; annotation files are stored in the table's results directory under `annotations`. There is no implemented config-file loader or resume mode. + +The `giant` CLI exposes only `annotate` and `extract`, each taking an exact trial-table filename. Its only microscope choice is `slap2` and its only scan-mode choice is `band`. Unsupported CLI choices are argument errors; unsupported routing through standalone services raises `ValueError`, not `NotImplementedError`. Pipeline operations delegate to those same services. For headless extraction using existing ROIs, select `--draw-user-rois --headless`. + +### Canonical API and ownership + +The Python API preceding this cleanup was unreleased and unused. Compatibility-only Python imports and adapters have been removed, not deprecated. MATLAB file-format compatibility is retained separately; there is no production Python API migration requirement. + +- `extract_band_sources(input, params=None, *, execution=None, annotations=None)` accepts an exact filename or loaded `TrialTable`. Use `Pipeline(..., params=...)` or `Pipeline.from_trial_table(..., params=...).extract()` for a session. +- `resolve_band_options(params=None, execution=None, annotations=None)` returns fresh validated `(BandSiloParams, ExecutionOptions, AnnotationOptions)` objects. Each argument accepts **only its corresponding dataclass, a dictionary of fields belonging to that scope, or `None`**. Mixed dictionaries, cross-scope fields and unknown keys are rejected; they are not redistributed or converted. Caller inputs are not mutated. +- `params` holds science and optional `num_channels`; `execution` holds `max_workers`, `verbose`, and `max_trials`; `annotations` holds `enabled`, `interactive`, and `operator`. Generic services and `Pipeline` take `microscope` / `scan_mode` as explicit routing options; only SLAP2 band extraction is implemented (`scan_mode=None` also selects band). There is no registration configuration class. +- All per-path results are accessed through `ExperimentSummary.paths`. A `PathSummary` owns `sources`, `user_rois`, `visualizations`, `frame_info`, `global_f`, and `z_depths`. There are no flat or aggregate summary properties. Access frame fields as attributes, such as `path.frame_info.offlineXshifts`, or call the shallow `FrameInfo.to_dict()`; `FrameInfo` is not a mapping. +- `giant_python.io.read_summary(path)` returns an `ExperimentSummary`; `giant_python.io.write_summary(summary, path)` accepts that model and returns `None`. Their owner is `giant_python.io.experiment_summary`; the model's `from_h5()` / `to_h5()` delegate there. No public assembled-dictionary writer or alternate summary IO alias is supported. +- `giant_python.extraction.band.result_assembly.assemble_path_summary()` is the band result assembly boundary and returns a `PathSummary`. Scientific backend code belongs in `giant_python.extraction.band`; shared kernels belong in `giant_python.numerics`, not the removed compatibility namespaces. +- `giant_python.gui.parameters_gui.run_parameter_gui(params=None, *, execution=None, annotations=None)` returns `(BandSiloParams, ExecutionOptions, AnnotationOptions)` or `None` on cancellation. It retains the split scopes and does not modify caller-owned options. +- `giant_python.io` exports the working `load_struct_h5`, `read_trial_table`, `read_summary`, `write_summary`, `read_annotations_h5`, `save_annotations_h5`, `load_alignment_data_h5`, `open_slap2_file`, `read_tiff` and `Slap2Reader`. `write_dict_to_h5group` remains a working helper in `giant_python.io.hdf5`, not a universal model serializer. `Slap2Reader` describes the vendor reader contract for typing; it is not an unimplemented workflow stage. + +The shared `gaussian_peaks_integrated()` API in `giant_python.numerics.peaks` accepts only `(N, 5)` parameters `[amp, mu_y, mu_x, sigma_y, sigma_x]`; equal widths must be supplied in both columns. Peak fitting includes an isotropic warm start followed by anisotropic refinement, using canonical five-column parameters with tied widths internally during the first stage. Removing the alternate parameter representation does not retune this fitting policy or the scientific defaults. + ## Epoch and Analysis Trial -For each experiment we run through the pipeline, we break down the data into epochs and analysis trials. +The following describes the reference GIAnT organization concepts. Python currently consumes these structures from an existing trial table rather than implementing the organization stage. Epochs are full experimental sessions that can be aligned with each other (i.e. the same field of view and regions of interest are being imaged). Analysis trials are generally contiguous subsets (in time) of an epoch. These analysis trials may not align exactly with experimental trials. For SLAP2, analysis trials are the experimental trials if the data was collected using the multi-trial functions of the SLAP2 and each trial is saved off the microscope in a different file. If data was continuously collected on SLAP2, the experiment will be split up into analysis trials of length 200000 lines (~20 sec) to help parallelize processing. -For data not collected on SLAP2, the current GIAnT pipeline sets Epochs to be 1 and each file that is selected to be processed is an analysis trial. These analysis trials must be able to be aligned to one another. +For data not collected on SLAP2, the reference GIAnT-MATLAB pipeline sets Epochs to be 1 and each file that is selected to be processed is an analysis trial. These analysis trials must be able to be aligned to one another. This is reference behavior, not a supported Python acquisition route. ## Pipeline Outputs -GIAnT-python writes the **same set of HDF5 output files with the same field names, shapes, dtypes, and conventions** as [GIAnT-MATLAB](https://github.com/AllenNeuralDynamics/GIAnT-MATLAB), so files produced by either toolbox are interchangeable. The schemas below are the on-disk contract; the typed data models in `giant_python.models` (`TrialTable`, `AlignmentData`, `ExperimentSummary`, ...) mirror these structures and own the `from_h5` / `to_h5` (de)serialization. +The extensive trees and field tables below are **reference schemas**, not a list of everything Python emits. The working BandSILo writer emits a subset of the experiment summary: source `profiles`, `coords`, `dF_ls`, `F0`, `SNR`; visualizations; `Z_depths`; global `F`; frame bookkeeping and shifts; and, when enabled, ROI `labels`, `mask`, and `F`. Disabled annotations omit the ROI group; enabled-but-empty annotations are distinct. + +**Future/reference-only output:** `dF_denoised`, `events`, ROI `Fsvd`, and the entire per-trial summary are **not emitted**. Fields retained on model classes do not imply that the band codec reads/writes them. Arbitrary reference files are not guaranteed to round-trip losslessly through `ExperimentSummary`. + +Trial-table writing is unavailable, and `TrialTable.to_h5` and the `AlignmentData` model have been removed. The shared HDF5 decoder and raw acquisition adapters nevertheless read the inputs needed by BandSILo. Existing summary writes replace supplied path groups but retain unrelated path groups and existing root parameters/layout flags; this is not an atomic replacement or resume mechanism. Prefer a fresh output location for a new analysis. ### Reading and writing H5 -GIAnT-python reads and writes these files with [`h5py`](https://www.h5py.org/), which stores arrays in **row-major (C) order**. Files written by GIAnT-python therefore set the `row_major` flag to `1`, and **every dimension tuple listed below matches the h5py `shape` directly** (i.e. read the tuples left-to-right as the axis order returned by `h5py`). +The implemented codecs use [`h5py`](https://www.h5py.org/). New Python band summary and annotation files set `row_major` to `1`: the logical dimensions below then match the dataset's h5py `shape`. Reference-only fields retain their intended logical dimensions for future ports; they are not all written by Python. Updating an existing summary preserves its existing layout flag, so use a fresh destination rather than mixing producers in place. -Files produced by GIAnT-**MATLAB** instead set `row_major` to `0` (or omit it), because MATLAB stores arrays column-major and its dimension tuples follow MATLAB `size()`. GIAnT-python's loaders inspect the `row_major` flag and **permute axes as needed**, so a MATLAB-written file and a Python-written file deserialize to the same logical arrays. When in doubt: +Files produced by GIAnT-**MATLAB** instead set `row_major` to `0` (or omit it), and their dimension tuples follow MATLAB `size()`. The shared decoder inspects this flag and **reverses multidimensional axes as needed** to recover logical orientation. Schema-specific support is still required; layout conversion does not imply support for every MATLAB file. When in doubt: | `row_major` value | Layout | Axis order of the tuples below | | --- | --- | --- | | `1` | row-major (h5py / NumPy C order) | matches h5py `shape` as written here | | `0` or absent | column-major (MATLAB `size()`) | reverse of the tuples here | -**Vector shapes are preserved on disk.** Fields documented as `1 x N` (row vectors) or `N x 1` (column vectors) are written as rank-2 datasets with those exact dimensions — e.g. `motionDSc` is shape `(1, nDSframes)`, not a squeezed `(nDSframes,)`. GIAnT-python does **not** collapse singleton dimensions when saving. Downstream code can therefore use the same size checks and indexing on MATLAB- or Python-generated files after handling the `row_major` flag (and any required axis permutation for multi-D arrays). +**Emitted vector shapes are preserved on disk.** Band summary frame vectors retain rank two, rather than being squeezed to `(nFrames,)`. For reference-only fields, the listed vector shapes describe the reference schema, not a Python writer guarantee. Layout handling alone does not establish full file interchangeability. ### Index and coordinate conventions -These conventions are shared with GIAnT-MATLAB and preserved on both read and write: +These conventions describe the supported subset and the reference fields below: - **Line/frame indices** (`first_line`, `last_line`, `DSframes`, `frame_line_idxs`) are **1-indexed** to retain the SLAP2 line-indexing convention. - **Spatial coordinate fields** in the summaries (`sources/spatial/coords`, `act_im_peaks`, `per_trial_*_coords`, etc.) are **0-indexed** `[z_loc, y_loc, x_loc]`, matching image axis order (`fastz`, rows, cols). `z_loc` is the 0-based index into the `fastz` axis. @@ -68,11 +154,15 @@ These conventions are shared with GIAnT-MATLAB and preserved on both read and wr - **Images** are stored `rows x cols` (row = Y, col = X). Motion shifts: `motionDSr` / `motionR` are **row** shifts, `motionDSc` / `motionC` are **column** shifts. - `params/activityChannel` is **1-indexed** into the recording's `numChannels` channels. +The last field is a reference parameter: do not assume every reference parameter is emitted by the band writer. The Python `BandSiloParams.activity_channel` is zero-based; currently only `0` is supported, and source activity arrays for other channels retain NaNs. + +**Motion-sign convention:** public `FrameInfo.offlineXshifts`, `offlineYshifts`, and `offlineZshifts` contain alignment displacement with the standard MATLAB SILo sign. Private band reference offsets have the opposite sign and are converted once during `assemble_path_summary()`, not by generic IO. If an experimental file from an unreleased Python checkout exists, establish its producer/alignment provenance before interpreting or changing its shifts; there is no known production Python migration to perform. Never negate MATLAB-produced or already-correct summaries again. Online shifts are unchanged. Matching this sign does not change the existing linear/clamped interpolation to MATLAB's PCHIP/extrapolation. Removing Python compatibility code leaves the HDF5 field spelling `Z_depths`, layout flags, coordinate conversions, scientific defaults, motion-bin ordering, and numerical call order unchanged; no new caching or execution policy is introduced. + Legend for the trees below (🗄️ file · 📁 group · 🔤 string · 🔢 integer · 📈 numeric · 🖼️ image · ☑️ bool). ### Trial Table -Each experiment processed with GIAnT first gets a `trial_table.h5` file that summarizes relevant file locations and analysis trial structures. The `slap2_info` group is only populated for SLAP2 experiments. The `motion_correction` and `source_extraction` groups are populated by downstream pipeline stages and will only be present once those stages have run. (Model: `giant_python.models.TrialTable` / `Slap2Info`.) +Reference input schema: a trial table summarizes file locations and analysis trial structures. Python loads an existing pre-registered table; it does not currently create/save this model or promise to append stage groups. `slap2_info` is SLAP2-specific; the reference pipeline populates `motion_correction` and `source_extraction` downstream. (Model: `giant_python.models.TrialTable` / `Slap2Info`.) ``` 🗄️ trial_table.h5 @@ -107,7 +197,7 @@ Each experiment processed with GIAnT first gets a `trial_table.h5` file that sum ### Alignment Data -The motion correction stage saves a H5 file ending in `_ALIGNMENTDATA.h5` that contains the alignment data for each trial. (Model: `giant_python.models.AlignmentData`.) +Reference input schema: the external motion correction stage saves an alignment HDF5 file for each trial. Python reads the band fields through `giant_python.io.slap2.load_alignment_data_h5`; there is no `AlignmentData` model or Python registration stage. The full external schema is retained below even where fields are not consumed by the band workflow. ``` 🗄️ _ALIGNMENTDATA.h5 @@ -177,8 +267,8 @@ The final step of the pipeline, source extraction (Source Identification by Acti ├ 📁 sources | ├ 📁 temporal | | ├ 📈 dF_ls (sources x channels x total frames) - | | ├ 📈 dF_denoised (sources x channels x total frames) - | | ├ 📈 events (sources x channels x total frames) + | | ├ 📈 dF_denoised (sources x channels x total frames; REFERENCE ONLY, not emitted) + | | ├ 📈 events (sources x channels x total frames; REFERENCE ONLY, not emitted) | | ├ 📈 F0 (sources x channels x total frames) | | └ 📈 SNR (sources x 1) | └ 📁 spatial @@ -187,7 +277,7 @@ The final step of the pipeline, source extraction (Source Identification by Acti ├ 📁 user_rois | ├ 🔤 labels (rois x 1) | ├ 🖼️ mask (rois x fastz x rows x cols) - | ├ 📈 Fsvd (rois x channels x total frames) + | ├ 📈 Fsvd (rois x channels x total frames; REFERENCE ONLY, not emitted) | └ 📈 F (rois x channels x total frames) ├ 📁 visualizations | ├ 🖼️ mean_im (channels x fastz x rows x cols) @@ -207,7 +297,7 @@ The final step of the pipeline, source extraction (Source Identification by Acti └ ☑️ discard_frames (total frames x 1) ``` -A summary file of per-trial data is also saved as `per_trial_summary.h5` for any fields that may vary across analysis trials. The trial axis matches `trial_table.h5` (all analysis trials); trials without alignment or source-extraction data are left as NaN in the corresponding slices. +**Reference only — not emitted by Python BandSILo.** The reference per-trial summary holds fields that vary across analysis trials. Its trial axis matches the reference trial table (all analysis trials); trials without alignment or source-extraction data are represented as NaN slices. The entire tree below is retained as external-format documentation, not as a Python implementation commitment. ``` 🗄️ per_trial_summary.h5 @@ -228,7 +318,7 @@ A summary file of per-trial data is also saved as `per_trial_summary.h5` for any ### Band Registration Lookup Table (intermediate file) -`BandRegistration` builds this file once per experiment under `motion_correction/bandRegLookupTable.h5` and reuses it on subsequent runs. XY search limits (`xPre`, `yPre`, etc.) are shared across paths; per-path superpixel and reference-stack data live under `Path{n}`. +External MATLAB `BandRegistration` builds this reference input under `motion_correction/bandRegLookupTable.h5`. Python BandSILo consumes it; Python registration and lookup-table generation are not implemented. XY search limits (`xPre`, `yPre`, etc.) are shared across paths; per-path superpixel and reference-stack data live under `Path{n}`. ``` 🗄️ bandRegLookupTable.h5 @@ -248,6 +338,8 @@ A summary file of per-trial data is also saved as `per_trial_summary.h5` for any ## File Field Descriptions +These tables retain the full reference fields for orientation and future ports. They do not override the implemented-subset limits above. In particular, registration outputs are inputs to Python, and all per-trial-summary fields are future-only. + ### `trial_table.h5` | Field | Size | Data type | Description | @@ -378,19 +470,19 @@ Top-level fields written by **all three** motion correction backends: `numChanne | `Path{n}/global/F` | channels x total frames | numeric | Fluorescence traces over the whole field (one row per channel) | | `Path{n}/user_rois/labels` | rois x 1 | string | User-defined ROI labels | | `Path{n}/user_rois/mask` | rois x fastz x rows x cols | uint8 or bool | Stacked binary masks for each user ROI | -| `Path{n}/user_rois/Fsvd` | rois x channels x total frames | numeric | ROI signals after SVD / projection step (if used) | +| `Path{n}/user_rois/Fsvd` | rois x channels x total frames | numeric | **Future/reference only; not emitted.** ROI signals after SVD / projection step | | `Path{n}/user_rois/F` | rois x channels x total frames | numeric | Raw or baseline-corrected ROI fluorescence | | `Path{n}/sources/spatial/profiles` | sources x fastz x rows x cols | numeric | Spatial component / pixel weights per source, averaged across trials with footprints | | `Path{n}/sources/spatial/coords` | sources x 3 | numeric | Source centers per row: `[z_loc, y_loc, x_loc]`, **0-indexed** | | `Path{n}/sources/temporal/dF_ls` | sources x channels x total frames | numeric | Least-squares ΔF (absolute or scaled) | -| `Path{n}/sources/temporal/dF_denoised` | sources x channels x total frames | numeric | Denoised ΔF | -| `Path{n}/sources/temporal/events` | sources x channels x total frames | numeric | Deconvolved source events | +| `Path{n}/sources/temporal/dF_denoised` | sources x channels x total frames | numeric | **Future/reference only; not emitted.** Denoised ΔF | +| `Path{n}/sources/temporal/events` | sources x channels x total frames | numeric | **Future/reference only; not emitted.** Deconvolved source events | | `Path{n}/sources/temporal/F0` | sources x channels x total frames | numeric | Baseline estimate used for normalization | | `Path{n}/sources/temporal/SNR` | sources x 1 | numeric | (optional) Signal-to-noise ratio metric; only written when extraction emits per-source SNR | ### `per_trial_summary.h5` -The trial axis matches `trial_table.h5` (all analysis trials); trials without alignment or source-extraction data are left as NaN in the corresponding slices. Coordinate fields use the same **0-indexed** `[z_loc, y_loc, x_loc]` convention as `experiment_summary.h5`; `z_loc` is currently always `0`. +**All fields in this table are future/reference only; this file is not emitted.** The reference trial axis matches the trial table (all analysis trials); trials without alignment or source-extraction data are left as NaN in the corresponding slices. Coordinate fields use **0-indexed** `[z_loc, y_loc, x_loc]` coordinates. | Field | Size | Data type | Description | | --- | --- | --- | --- | diff --git a/docs/source/architecture.rst b/docs/source/architecture.rst new file mode 100644 index 0000000..994d055 --- /dev/null +++ b/docs/source/architecture.rst @@ -0,0 +1,200 @@ +Architecture and ownership +========================== + +Scope +----- + +The working path starts with existing registered SLAP2 band inputs and ends +with annotation and BandSILo extraction. Standard SILo, organization, +registration, deconvolution, config loading and checkpoint/resume are not +implemented. As of **2026-09-20**, the earlier requirement to retain future +placeholders is superseded: unimplemented placeholder files, functions and +exports have been removed rather than maintained as future contracts. +MATLAB external-format schemas remain documented separately from Python API +availability; see the repository README and :doc:`migration`. + +Directory owners +---------------- + +All paths below are relative to ``src/giant_python``. + +.. list-table:: Canonical responsibilities + :header-rows: 1 + :widths: 24 76 + + * - Owner + - Responsibility + * - ``pipeline/`` + - Public extraction/annotation services, explicit capability routing, + and the chainable ``Pipeline``. Only annotation and extraction are + supported. The CLI and standalone calls use these same services. + * - ``models/`` + - Portable metadata, split configuration and eager public results. + Convenience codecs delegate to shared IO; construction does not + perform band file discovery or motion normalization. + * - ``io/`` + - Shared HDF5 decoding, trial-table reading, emitted experiment-summary + codecs, raw annotation persistence and acquisition adapters. Schema + reading must not silently change the sign of alignment displacement. + * - ``extraction/band/`` + - Working scientific backend: input preparation, geometry, motion bins, + operators, background, activity, localization, summary images, trial + data reduction, traces, ROI geometry and result assembly. ``workflow`` + composes these stages. ``inputs`` owns runtime path resolution, + private motion normalization, lookup/reference/PSF loading and asset + discovery. Pure pixel support and PSF padding/cropping belong to + ``geometry``; numerical assembly belongs to ``result_assembly``. + TIFF decoding stays in ``io.tiff``. + * - ``numerics/`` + - Acquisition-independent baseline, peaks, interpolation and morphology. + Generic registration/deconvolution/NMF/variance stubs were removed; + the working band-specific NMF solver remains in ``localization``. + * - ``gui/`` + - Presentation: the existing ROI editor and parameter form. OpenCV and + tkinter are loaded for interactive work, not model deserialization. + Napari is not the implemented band editor. + * - ``parallel.py`` / ``progress.py`` + - Shared ordered trial mapping and opt-in logging/progress. ``tqdm`` is + imported lazily for verbose progress and is a declared dependency. + * - ``assets/psfs/`` + - Bundled PSF templates; no asset relocation or new cache is required. + +The former Python API was unreleased and unused. Compatibility-only +``bandsilo`` / ``math`` imports and extraction adapters are removed, not +alternative owners or supported aliases. MATLAB file-format support is a +separate IO responsibility. See :doc:`migration` for the canonical owner map. + +Input identity and configuration +-------------------------------- + +``extract_band_sources(input, params=None, *, execution=None, annotations=None)`` +accepts an exact filename or a loaded ``TrialTable``. +``Pipeline.from_trial_table`` loads a filename once and retains it as +``trial_table_path``; a loaded table is retained directly. ``source_path`` is +provenance, not a filename reconstructed from ``savedr``. Runtime input +preparation derives raw and alignment locations from the model's explicit +``datadr`` and ``savedr``. Copying the table itself does not relocate them. + +``BandSiloParams`` owns scientific settings, ``ExecutionOptions`` owns worker +count, verbose reporting and the optional trial limit, and +``AnnotationOptions`` owns ROI inclusion, interactivity and operator metadata. +Resolution does not mutate the caller's options to store inferred channel +counts. ``ResolvedAcquisition`` carries resolved backend metadata separately. +Annotation-disabled and enabled-but-empty are distinct states. + +``Pipeline(..., params=...)`` and ``Pipeline.from_trial_table(..., params=...)`` +use the same split options. ``resolve_band_options`` accepts only the matching +dataclass, a dictionary of fields belonging to that scope, or ``None`` for +each argument, and returns fresh validated objects. It does not accept mixed +configuration dictionaries or redistribute routing/run/ROI fields from science. +Routing belongs to explicit ``microscope`` / ``scan_mode`` arguments on the +generic services and ``Pipeline``. Only ``microscope='slap2'`` and band mode +are supported; ``scan_mode=None`` also selects band. Standalone routing checks +raise ``ValueError`` before loading a backend or input, not +``NotImplementedError``. ``AlignParams`` has been removed. + +The public workflow methods on ``Pipeline`` are ``from_trial_table``, +``annotate`` and ``extract``. The constructor accepts ``microscope``, ``params``, +``execution``, ``annotations`` and ``scan_mode``; ``save_dir`` and ``align_params`` +options/attributes no longer exist. Output location comes from ``tt.savedr``. +There are no ``from_config``, ``organize``, ``register`` or ``run_all`` methods, +nor generic stage/builder/corrector classes. The ``giant`` CLI exposes only +``annotate`` and ``extract``, with ``slap2`` / ``band`` as the only routing +choices; invalid commands/options are argument errors. + +``gui.parameters_gui.run_parameter_gui(params=None, *, execution=None, +annotations=None)`` uses these same scopes and returns their validated triple +or ``None`` on cancellation. The existing form, including its natural-log +sparse-factor control, is retained without a combined parameter carrier. + +Typed results and array ownership +--------------------------------- + +``ExperimentSummary.paths`` contains ``PathSummary`` records. Each path owns +its sources, user ROIs, visualizations, depths, global fluorescence and typed +``FrameInfo``. ``path_index`` is **zero-based**; HDF5 names are **one-based**: +``path_index=0`` maps to ``Path1``. Do not infer identity from a path's list +position when reading a subset of saved paths. + +``Source`` and ``UserRoi`` entries can be views of assembled arrays, sharing +data. ``FrameInfo.to_dict()`` is shallow. These are **eager NumPy-backed +results**, not lazy HDF5 proxies, immutable snapshots, or independent copies. +Mutating shared storage can affect other views. Persistence may stack +per-source arrays; the API does not promise zero-copy serialization. + +``ExperimentSummary.paths`` is the only per-path result interface; there are +no flat convenience or aggregate source/ROI properties. ``FrameInfo`` exposes +attributes and ``to_dict()``, not mapping methods or a mapping adapter. + +Inside the backend, ``TrialTraceResult`` names the historical eight-tuple +fields without changing their order. Source traces are frames by sources; +global fluorescence is frames by channels, and ROI traces are frames by ROIs +by channels. Its ``reference_offsets`` are private row/column/Z offsets, +not the public saved displacement. ``PathResult`` collects numerical outputs +before their public summary assembly. + +``result_assembly.assemble_path_summary(PathResult, path_index=0)`` directly +constructs a ``PathSummary``. It is the only band result assembly interface; the former +assembled-dictionary entry point and writer adapters are removed. Private +schema dictionaries inside IO remain implementation details, not a second +public result API. + +Persistence boundaries +---------------------- + +The shared experiment codec supports the **emitted band subset**, not every +field in the README's reference schemas. Future ``dF_denoised``, ``events``, +``Fsvd`` and per-trial summaries are not emitted. Model fields reserved for +those concepts do not establish codec support. Trial-table saving and the +``AlignmentData`` model codecs are unavailable: ``TrialTable.to_h5`` and the +alignment model were removed. Raw alignment reading is implemented separately +by ``io.slap2.load_alignment_data_h5`` and retains saved displacement signs. + +``io.hdf5.load_struct_h5`` and ``write_dict_to_h5group`` are working decoder and +parameter-group writer helpers. Removing the unimplemented ``save_struct_h5`` +does not remove HDF5 writing, annotation persistence or the summary codec. +Native ``io.tiff.read_tiff`` and ``io.slap2.open_slap2_file`` remain; the generic +ScanImage wrappers and ``get_online_motion`` stub do not. ``Slap2Reader`` remains +a structural typing protocol for the vendor reader's existing attributes and +batched ``getLineData`` method; it promises neither a context manager nor a +close method and is not an executable placeholder stage. + +``io.experiment_summary.read_summary(path)`` returns ``ExperimentSummary``; +``write_summary(summary, path)`` accepts that model and returns ``None``. +Both are also exported by ``giant_python.io``. Model convenience methods +delegate to these functions. There are no alternate summary codec aliases or +public assembled-dictionary writers. + +The decoder honors ``row_major`` (absent or zero means MATLAB layout) and +reverses multidimensional axis order when needed. The band writer preserves +rank-two frame vectors and writes new files with ``row_major=1``. Numeric +line indexes remain one-based; output spatial coordinates are zero-based +``[z, y, x]``. Annotation coordinate flags distinguish current zero-based +``[y, x]`` from legacy MATLAB one-based ``[x, y]``. +The HDF5 depth field remains exactly ``Z_depths``; its model attribute is +``PathSummary.z_depths``. These format and coordinate conventions are unaffected +by removal of Python compatibility code. + +Writing an existing summary retains root parameters/layout flags and unrelated +paths, while replacing supplied path groups. This existing behavior is not +transactional output, a fresh-file guarantee, or resume support. Unknown +reference fields are not guaranteed a lossless model round-trip. + +Scientific invariants +--------------------- + +Removing compatibility code and placeholders does not retune defaults, remove +the working band NMF solver, reorder kernels/motion bins/random calls/trials, +change interpolation or scheduling, add caching, or introduce lazy loading. +The public offline-motion contract established by the architecture refactor +matches standard MATLAB SILo: internal reference offsets become alignment +displacement once in ``assemble_path_summary``. IO does not apply another +conversion; online shifts are unchanged. See :doc:`migration` for sign and +provenance cautions, not a production Python migration procedure. + +Shared Gaussian peak evaluation accepts only five-column parameters +``[amp, mu_y, mu_x, sigma_y, sigma_x]``. Equal widths must be explicit in both +columns. Peak fitting retains the implemented isotropic warm start followed +by anisotropic refinement: the first stage ties the two width columns +internally while keeping the canonical five-column representation. Removing +four-column compatibility is not a change to thresholds or fitting policy. \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst index 07adcad..4830f88 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,17 +1,193 @@ -.. Doc Template documentation master file, created by - sphinx-quickstart on Wed Aug 17 15:36:32 2022. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. +GIAnT-Python +============ - -Welcome to this repository's documentation! -=========================================== +The implemented workflow is SLAP2 BandSILo extraction and ROI annotation +from a **pre-registered trial table**. Organization, registration, standard +SILo and deconvolution are not implemented; their placeholders have been removed. +Existing raw recordings, reference TIFFs, alignment files and the band +registration lookup table are required; this is not a full GIAnT-MATLAB +pipeline replacement. .. toctree:: :maxdepth: 2 :caption: Contents: - modules + architecture + migration + +Standalone extraction +--------------------- + +Both an exact trial-table filename and a loaded ``TrialTable`` are accepted. +The filename is never guessed from the results directory. For example, given +``table_or_filename`` supplied by the caller: + +.. code-block:: python + + from giant_python import ( + AnnotationOptions, BandSiloParams, ExecutionOptions, + extract_band_sources, + ) + + summary = extract_band_sources( + table_or_filename, + BandSiloParams(), + execution=ExecutionOptions(max_workers=6, verbose=True), + annotations=AnnotationOptions(enabled=False, interactive=False), + ) + +For a chainable session using the same extraction service: + +.. code-block:: python + + from giant_python import ( + AnnotationOptions, BandSiloParams, ExecutionOptions, Pipeline, + ) + + pipeline = Pipeline.from_trial_table( + table_or_filename, + params=BandSiloParams(), + execution=ExecutionOptions(max_workers=6), + annotations=AnnotationOptions(enabled=False, interactive=False), + ).extract() + summary = pipeline.summary + +The band signature is +``extract_band_sources(input, params=None, *, execution=None, annotations=None)``. +``Pipeline`` also uses ``params=...`` for science. Each option scope accepts +only its corresponding dataclass, a dictionary of its own fields, or ``None``. +``resolve_band_options`` returns a fresh validated triple; mixed dictionaries +and cross-scope fields are rejected, not translated into other option scopes. +Explicit ``microscope`` / ``scan_mode`` routing belongs to the generic services +and ``Pipeline``; only SLAP2 band is implemented (``scan_mode=None`` also selects +band). Unsupported standalone routing raises ``ValueError``, not +``NotImplementedError``. + +``Pipeline`` exposes ``from_trial_table``, ``annotate`` and ``extract``; +``from_config``, ``organize``, ``register`` and ``run_all`` have been removed. +There are no constructor options/attributes named ``save_dir`` or +``align_params``. Directory overrides belong on the loaded trial table. +The ``giant`` CLI has only ``annotate`` and ``extract`` subcommands, with +``--microscope slap2`` and ``--scan-mode band`` as its sole routing choices. +Unsupported commands or choices fail argument parsing. + +An explicit ``annotate_rois(...)`` call or ``Pipeline.annotate()`` loads +existing annotations or draws them when permitted. For headless extraction +with existing ROIs, use ``AnnotationOptions(enabled=True, interactive=False)``; +missing required annotations fail rather than silently disabling ROIs. + +Output is written beneath the loaded table's ``savedr`` in +``source_extraction/experiment_summary.h5``. Copying a table does not rewrite +its stored directories. The repository example ``examples/extract_band.py`` +stages inputs and overrides ``datadr``/``savedr`` on a loaded model explicitly. +It refuses existing output directories and never changes the environment at +import time. See :doc:`migration` for canonical owners, data conventions and +deployment notes. The preceding Python API was unreleased and unused; +compatibility-only imports and adapters are removed, not supported aliases. + +Results and parameter editing +----------------------------- + +Use ``summary.paths`` exclusively for per-path data. Each ``PathSummary`` has +its own sources, ROIs, visualizations, frame information, depths and global +fluorescence. ``path_index`` is zero-based; disk groups begin at ``Path1``. +Access ``FrameInfo`` fields as attributes or use its shallow ``to_dict()``; +there is no mapping adapter or flat summary convenience API. + +``giant_python.gui.parameters_gui.run_parameter_gui(params=None, *, execution=None, +annotations=None)`` returns a validated ``(BandSiloParams, ExecutionOptions, +AnnotationOptions)`` triple, or ``None`` on cancellation, without mutating inputs. +Toolkit imports remain lazy. + +Supported schema subset +----------------------- + +The repository README preserves extensive MATLAB reference trees and field +tables. They are reference material, **not** a claim that Python emits every +listed field. Band output includes source profiles/coordinates, ``dF_ls``, +``F0``, ``SNR``, visualizations, depths, global fluorescence, frame information +and optional user ROI masks/labels/``F``. + +``dF_denoised``, ``events``, ROI ``Fsvd`` and the entire +``per_trial_summary.h5`` are future/reference-only and **not emitted**. +``ExperimentSummary.from_h5`` and ``to_h5`` support the emitted band subset +eagerly. Generic HDF5 decoding and parameter-group writing are shared working +helpers. ``TrialTable.to_h5`` and the ``AlignmentData`` model have been removed; +the raw band alignment reader remains implemented. Native TIFF/SLAP2 readers, +annotation persistence and summary writing are also retained. + +The summary codec is model-only: ``giant_python.io.read_summary(path)`` returns +an ``ExperimentSummary`` and ``giant_python.io.write_summary(summary, path)`` +returns ``None``. Band assembly uses +``giant_python.extraction.band.result_assembly.assemble_path_summary`` to construct +``PathSummary`` directly; no public assembled-dictionary writer remains. +Removing Python compatibility surfaces does not change MATLAB layout handling, +the exact ``Z_depths`` field spelling, coordinate conversions or shift signs. + +Placeholder removal — 2026-09-20 +-------------------------------- + +The earlier requirement to retain future placeholders is superseded. All +unimplemented placeholder modules/functions and corresponding exports have +been removed, including ``AlignParams``, future pipeline stages, generic +numerical stubs and unsupported IO wrappers. Working band science, peak fitting, +the band-specific NMF solver and HDF5 codecs are unchanged. ``Slap2Reader`` is +retained as a typing protocol for real vendor readers, not a placeholder stage. +The README's MATLAB schemas remain external-format reference documentation. +See :doc:`migration` for the removal inventory and :doc:`architecture` for +current owners. + +Public API reference +-------------------- + +These modules expose models, canonical IO and lazy workflow services. +Autodoc deliberately does not traverse the numerical backend or GUI toolkit +modules. + +.. automodule:: giant_python.pipeline.extract + :members: extract_band_sources, extract_sources + +.. automodule:: giant_python.pipeline.annotate + :members: annotate_rois + +.. autoclass:: giant_python.pipeline.pipeline.Pipeline + :members: from_trial_table, annotate, extract + +.. automodule:: giant_python.models.params + :members: BandSiloParams, ExecutionOptions, AnnotationOptions, resolve_band_options + +.. autodata:: giant_python.models.params.BandParamsInput + +.. autodata:: giant_python.models.params.ExecutionInput + +.. autodata:: giant_python.models.params.AnnotationInput + +.. autodata:: giant_python.pipeline.extract.TrialTableInput + +.. automodule:: giant_python.models.experiment_summary + :members: ExperimentSummary, PathSummary, FrameInfo, Source, UserRoi, Visualizations + +.. automodule:: giant_python.io.experiment_summary + :members: read_summary, write_summary + +.. autoclass:: giant_python.models.trial_table.TrialTable + :members: from_h5 + +.. automodule:: giant_python.io.slap2 + :members: Slap2Reader, open_slap2_file, load_alignment_data_h5 + +.. automodule:: giant_python.io.tiff + :members: read_tiff + +.. automodule:: giant_python.io.hdf5 + :members: load_struct_h5, write_dict_to_h5group + +The package export lists below include only the current public surfaces; +removed placeholder names are not aliases or deprecated entry points. + +.. autodata:: giant_python.__all__ + +.. autodata:: giant_python.io.__all__ Indices and tables diff --git a/docs/source/migration.rst b/docs/source/migration.rst new file mode 100644 index 0000000..be28bcb --- /dev/null +++ b/docs/source/migration.rst @@ -0,0 +1,301 @@ +API and data conventions +======================== + +Public workflow and options +--------------------------- + +Use the lazy package-root exports ``extract_band_sources``, +``extract_sources``, ``annotate_rois``, ``Pipeline``, ``TrialTable``, +``ExperimentSummary``, ``BandSiloParams``, ``ExecutionOptions`` and +``AnnotationOptions``. Both extraction functions accept an exact trial-table +filename or a loaded model. Do not reconstruct a filename from ``savedr``. +``Pipeline.from_trial_table(..., params=BandSiloParams()).extract()`` returns +the same pipeline; read the extraction result through ``pipeline.summary``. + +The band service signature is +``extract_band_sources(input, params=None, *, execution=None, annotations=None)``. +``Pipeline(..., params=...)`` uses the same scientific option scope. The chain +method is ``extract()``; ``extract_sources()`` is a standalone generic service, +not a second chain method. Generic extraction and annotation services, and +``Pipeline``, take explicit ``microscope`` / ``scan_mode`` routing arguments; +only ``microscope='slap2'`` with band mode is implemented. ``scan_mode=None`` +also selects band. Unsupported routing through standalone services raises +``ValueError`` before input/backend loading, not ``NotImplementedError``. + +``Pipeline`` has only ``from_trial_table``, ``annotate`` and ``extract`` as its +public workflow methods. ``from_config``, ``organize``, ``register`` and +``run_all`` are removed, together with constructor options/attributes +``save_dir`` and ``align_params``. Use the loaded table's ``datadr`` / ``savedr`` +for explicit input/output directory overrides. Missing loaded state raises +``ValueError`` when an operation is requested. + +The ``giant`` CLI offers only ``annotate`` and ``extract``, each with a trial-table +filename. Its only microscope choice is ``slap2`` and its only scan-mode choice +is ``band``. Unsupported commands/options are argument errors (exit code 2). +Use ``--draw-user-rois --headless`` to extract with existing annotations without +drawing; missing required annotations fail explicitly. + +.. list-table:: Independent configuration scopes + :header-rows: 1 + + * - Argument + - Dataclass + - Scope + * - ``params`` + - ``BandSiloParams`` + - Band science and optional ``num_channels`` override + * - ``execution`` + - ``ExecutionOptions`` + - ``max_workers``, ``verbose``, ``max_trials`` + * - ``annotations`` + - ``AnnotationOptions`` + - ``enabled``, ``interactive``, ``operator`` + +``resolve_band_options(params=None, execution=None, annotations=None)`` returns +fresh validated objects in the order shown above. Each input accepts **only +its corresponding dataclass, a dictionary of that scope's fields, or None**. +Mixed dictionaries, cross-scope fields, unknown keys and unrelated dataclasses +are rejected; fields are not moved between scopes or converted from a combined +carrier. Caller-owned inputs are not mutated, even when acquisition channel +counts or GUI policy are later resolved. + +Defaults remain peak threshold 7, analysis rate 100 Hz, decay 0.15 seconds, +baseline window 4 seconds, denoise window 1 second, VIF 1.38 and PSF dilation 17. +Execution defaults to six workers, no trial limit and verbosity off. Annotation +defaults to disabled with automatic interactivity and operator ``SLAP2 User``. +``BandSiloParams.activity_channel`` is zero-based and only ``0`` is implemented. +``AlignParams`` has been removed; registration has no Python configuration API. + +Canonical owner map +------------------- + +The preceding Python API was unreleased and unused. Compatibility-only +namespaces, combined options, extraction adapters and result/IO aliases have +been removed, not deprecated. No old Python import support or production +Python migration procedure is promised. MATLAB file-format compatibility is +retained independently of those removed Python surfaces. + +.. list-table:: Canonical modules under giant_python + :header-rows: 1 + :widths: 44 56 + + * - Owner + - Interface or responsibility + * - ``pipeline.extract`` / ``pipeline.annotate`` / ``pipeline.pipeline`` + - Root-exported extraction/annotation functions and ``Pipeline``; + common services also used by the CLI + * - ``models.params`` + - Split configuration and ``resolve_band_options``; no registration options + * - ``models.experiment_summary`` + - ``ExperimentSummary``, ``PathSummary``, ``FrameInfo``, ``Source``, + ``UserRoi`` and ``Visualizations`` + * - ``io.experiment_summary`` + - ``read_summary(path)`` / ``write_summary(summary, path)``; + also exported by ``io``; model-only summary interface + * - ``io.hdf5`` / ``io.trial_table`` + - ``load_struct_h5`` and parameter-group writing; ``read_trial_table`` + builds portable metadata and serves ``TrialTable.from_h5`` + * - ``io.slap2`` / ``io.tiff`` + - Raw acquisition/alignment reading and TIFF decoding, without private + band motion-sign normalization + * - ``extraction.band.inputs`` + - Runtime paths, kept trials, private reference offsets, + lookup/reference loading and PSF discovery (including ``default_psf``) + * - ``extraction.band.workflow`` / ``extraction.band.result_assembly`` + - Scientific orchestration; ``assemble_path_summary`` constructs the + public path model, with persistence delegated to shared IO + * - ``extraction.band.geometry`` / ``extraction.band.operators`` + - Pure pixel support, PSF padding/cropping and sparse projection operators + * - ``extraction.band.motion_binning`` / ``extraction.band.background`` / + ``extraction.band.activity`` + - Band motion policies, background estimation and activity calculations + * - ``extraction.band.localization`` + - Implemented band-specific NMF/localization; the generic NMF stub is removed + * - ``extraction.band.traces`` / ``extraction.band.summary_images`` / + ``extraction.band.trial_data`` + - Band trace extraction, image assembly and superpixel data reduction + * - ``numerics.baseline`` / ``numerics.peaks`` / ``numerics.interpolation`` / + ``numerics.morphology`` + - Shared implemented numerical kernels and helpers + * - ``gui.roi_editor`` / ``gui.parameters_gui`` + - Interactive presentation with lazy toolkit imports + * - ``io.annotations`` / ``extraction.band.rois`` / ``extraction.band.annotation`` + - Annotation persistence, pure band ROI geometry and annotation policy + * - ``parallel`` / ``progress`` + - Ordered trial mapping and opt-in logging/progress + +Placeholder removal — 2026-09-20 +-------------------------------- + +The earlier requirement to retain future placeholders is explicitly superseded. +Unimplemented functions/files and corresponding exports are removed, not +deprecated or replaced by forwarding modules. This is an API-scope cleanup, +not an implementation of the omitted GIAnT-MATLAB stages. + +Removed modules under ``giant_python``: + +* ``numerics.deconv``, ``numerics.nmf``, ``numerics.registration`` and + ``numerics.variance`` (generic numerical stubs). +* ``pipeline.base``, ``pipeline.organize`` and ``pipeline.register`` (including + ``Stage``, ``TrialTableBuilder`` and ``MotionCorrector``). +* ``gui.draw_rois`` (not the working ROI editor) and ``models.alignment`` + (including ``AlignmentData``). + +Removed members in retained modules include ``AlignParams``, +``TrialTable.to_h5``, ``numerics.interpolation.interp_frame``, +``io.hdf5.save_struct_h5``, ``io.slap2.get_online_motion``, +``io.tiff.scanimagetiff_wrapper``, ``io.tiff.scanimagetiff_data_wrapper``, +and ``pipeline.extract.silo``. +The unsupported Pipeline methods/options listed above and their related exports +are also gone. Config-file construction, checkpoint/resume, organization, +registration, standard SILo, deconvolution and trial-table writing remain +unavailable; calling a removed API is not a supported capability check. + +Retained functionality includes ``nearest_interp``, peak detection/fitting, +baseline/morphology kernels, the working band NMF/localization solver, native +TIFF/SLAP2 readers, raw alignment decoding, ``write_dict_to_h5group``, annotation +persistence and experiment-summary codecs. ``Slap2Reader`` is a structural +typing protocol for real vendor readers, not a workflow stub to delete. +Scientific algorithms, defaults, ordering and external file conventions are +unchanged by placeholder removal. The README retains full MATLAB schema +descriptions, including unsupported fields and external registration outputs, +as reference documentation rather than promised Python functionality. + +Result, IO and GUI contracts +------------------------------ + +All imaging results belong to ``ExperimentSummary.paths``. Each ``PathSummary`` +owns its sources, ROIs, visualizations, ``FrameInfo``, depths and global +fluorescence. There are no flat or aggregate summary properties. ``path_index`` +is zero-based and maps to disk group ``Path{path_index + 1}``; a list position +is not necessarily that identity when reading selected paths. + +``FrameInfo`` exposes attributes and a shallow ``to_dict()`` only, not mapping +methods. Results are eager; per-source/ROI entries may share array storage. +They are neither lazy HDF5 proxies nor independent immutable snapshots. +Disabled annotations omit the ROI group; enabled-but-empty is a distinct state. + +``io.experiment_summary.read_summary(path)`` returns ``ExperimentSummary`` and +``write_summary(summary, path)`` accepts that model and returns ``None``. +``ExperimentSummary.from_h5`` / ``to_h5`` delegate to these canonical functions. +``extraction.band.result_assembly.assemble_path_summary`` directly builds +``PathSummary``; no public assembled-dictionary result adapter or writer remains. + +The codec supports the emitted band subset, not the full reference schema. +It does not emit denoised traces, events, ROI SVD fluorescence or per-trial +summaries. New files set ``row_major=1``. MATLAB layout decoding remains +supported (missing or zero flag reverses multidimensional axes as needed). +Rank-two frame vectors, one-based acquisition line indexes, zero-based spatial +coordinates and annotation coordinate flags are unchanged. The HDF5 field +spelling is exactly ``Z_depths``; ``z_depths`` is the path model attribute. + +Writing an existing summary replaces supplied path groups while retaining +unrelated paths and existing root parameters/layout flags. Required source/ROI +arrays are prepared before replacing each path, but the operation is not a +file-wide transaction or resume mechanism. Prefer a fresh destination; +unknown reference fields are not guaranteed a lossless model round-trip. + +``gui.parameters_gui.run_parameter_gui(params=None, *, execution=None, +annotations=None)`` returns ``(BandSiloParams, ExecutionOptions, +AnnotationOptions)`` or ``None`` on cancellation. The form retains its +natural-log sparse-factor control and returns validated options without +mutating inputs; it does not combine them into a legacy carrier. + +Gaussian parameter convention +----------------------------- + +``numerics.peaks.gaussian_peaks_integrated`` accepts only ``(N, 5)`` parameters +``[amp, mu_y, mu_x, sigma_y, sigma_x]``. Supply both widths explicitly, even +for isotropic Gaussians; there is no four-column adapter. +Peak fitting includes an isotropic warm start followed by anisotropic +refinement. Both stages retain five-column parameters; the first stage ties +width updates internally using their combined derivative, then the second +allows independent widths. Removing the alternate representation does not +retune scientific defaults or this implemented fitting policy. + +Motion sign and data provenance +------------------------------- + +Public ``FrameInfo.offlineXshifts``, ``offlineYshifts`` and +``offlineZshifts`` contain **alignment displacement**, matching standard +MATLAB SILo's saved sign. At matching sample locations this is the sign of +the original alignment ``motionDSc``, ``motionDSr`` and ``motionDSz``. +Displacement is added to reference/view-grid coordinates when sampling raw +images; mapping raw measurement positions into reference coordinates uses +its negative. These two mapping directions must not be conflated. + +The raw alignment reader preserves saved values. Band input normalization +negates offline displacement into private reference offsets. Band result +assembly in ``assemble_path_summary`` converts those offsets back exactly once +before building public ``PathSummary``/``FrameInfo`` objects or saving them. +The summary codec itself does not flip signs. No inter-trial XY correction is +invented where the band workflow has not computed one. + +The preceding Python version was never released or used, so no production +Python summary migration is required. **If an experimental file from an +unreleased checkout exists**, its offline sign needs producer/alignment-data +provenance. Do not auto-correct on read, use the installed package version +alone to infer file history, or infer provenance from zero-valued shifts. +Retain originals before any explicit transformation. MATLAB-produced and +already correct summaries must not be negated again. Online shifts are +unchanged by the Python API cleanup. + +This is sign compatibility, not a claim of identical interpolation: the band +code keeps its existing linear/clamped interpolation, whereas MATLAB's +standard summary builder uses PCHIP/extrapolation. Scientific defaults, +center/surround kernels, motion-bin ordering, numerical/random call order, +trial ordering, and internal extraction calculations are unchanged. + +Staging example +--------------- + +The repository's ``examples/extract_band.py`` is safe to import: +there are no import-time filesystem operations, environment assignments, +matplotlib imports or heavy GIAnT imports. + +Provide ``--trial-table``, ``--data-dir``, ``--motion-correction`` and a **new** +``--results-dir``. The example copies the exact supplied table and copies +existing motion-correction assets by default. ``--stage-motion symlink`` +selects directory linking (Windows may require Developer Mode or link +privileges). ``--stage-data copy`` or ``symlink`` stages the complete raw +directory, including sidecars and reference TIFFs; ``none`` uses it in place. +This is staging of externally generated motion correction, not execution of +Python registration (which is not implemented and has no placeholder API). + +Crucially, the example overrides ``datadr`` and ``savedr`` on the loaded +``TrialTable`` and passes that model directly to extraction. It does not +reload the copied file and lose those overrides. The copied file is an +unaltered provenance artifact: its stored directories are not rewritten, +and there is no ``TrialTable.to_h5`` method to call. Input filename fields +must be relative without parent traversal; absolute names are rejected rather +than silently bypassing staging or being guessed from basenames. + +The example refuses any existing results file, directory or symlink, and +refuses output nested inside an input tree. It never deletes arbitrary +destinations or silently retries by replacing a link. Partial output after +failure is left for inspection; choose a new directory for another run. +Copied source symlinks are retained, so verify their targets when relocating +data between machines. Symlink mode is intentionally not a portable copy. + +``--joblib-temp`` optionally selects an existing directory with sufficient +space for worker memory maps. Paths, loaded metadata and scientific/run +options are validated before any staging or environment changes. Its +environment override is scoped to staging/extraction, before importing the +lazy extraction service, and restored even when extraction fails. Without +the flag, the existing environment and joblib defaults are left alone. +No fixed container paths or operator names are embedded. Dependencies must +already be installed; the example does not install packages. + +Scientific and execution flags are explicit run choices, not changes to +library defaults. Use ``--use-rois --annotations-dir`` for existing headless +annotations, or add ``--interactive`` to permit drawing. Inspect ``--help`` +for all flags. + +Dependency note +--------------- + +Verbose progress imports ``tqdm`` lazily with no missing-package fallback; +it is declared directly. The existing band editor uses OpenCV and tkinter, +not napari. Removing Python compatibility code is not a GUI redesign and +changes no numerical dependency, quality threshold, coverage exclusion or +execution backend. \ No newline at end of file diff --git a/examples/extract_band.py b/examples/extract_band.py new file mode 100644 index 0000000..da2ea5b --- /dev/null +++ b/examples/extract_band.py @@ -0,0 +1,320 @@ +"""Extract registered band data, optionally staging inputs in a fresh run. + +Use --help for arguments. No files, environment variables, GUI toolkits, or +GIAnT modules are touched on import. This example copies existing motion +correction; it does not run the unimplemented Python registration stage. + +The source table must use relative recording/alignment filenames. Include +raw acquisition sidecars and reference TIFFs in --data-dir. A copied table +retains its original on-disk metadata: effective directory overrides are +made on the loaded model, not saved with the unsupported TrialTable writer. +""" + +from __future__ import annotations + +import argparse +import os +import shutil +from contextlib import contextmanager +from dataclasses import dataclass, replace +from pathlib import Path, PurePosixPath, PureWindowsPath +from typing import TYPE_CHECKING, Generator + +if TYPE_CHECKING: + from giant_python import ( + AnnotationOptions, + BandSiloParams, + ExecutionOptions, + TrialTable, + ) + + +@dataclass(frozen=True) +class InputPaths: + """Resolved input roots and a fresh destination, before any staging.""" + + table: Path + data: Path + motion: Path + results: Path + annotations: Path | None + joblib_temp: Path | None + + +@dataclass(frozen=True) +class PreparedRun: + """Validated paths, loaded metadata and independent run settings.""" + + paths: InputPaths + table: TrialTable + params: BandSiloParams + execution: ExecutionOptions + annotations: AnnotationOptions + + +def build_parser() -> argparse.ArgumentParser: + """Describe explicit input roots, safe staging, and split run options.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--trial-table", type=Path, required=True) + parser.add_argument( + "--data-dir", + type=Path, + required=True, + help="Raw recordings, sidecars, and reference TIFF directory", + ) + parser.add_argument( + "--motion-correction", + type=Path, + required=True, + help="Existing alignment and band registration lookup directory", + ) + parser.add_argument( + "--results-dir", + type=Path, + required=True, + help="New directory; an existing file, directory, or link is refused", + ) + parser.add_argument( + "--stage-data", + choices=("none", "copy", "symlink"), + default="none", + help="Keep raw inputs in place (default), copy, or link the full tree", + ) + parser.add_argument( + "--stage-motion", + choices=("copy", "symlink"), + default="copy", + help="Copy existing registration assets (default), or link them", + ) + parser.add_argument( + "--annotations-dir", + type=Path, + help="Optional existing annotation directory to copy into the run", + ) + parser.add_argument( + "--joblib-temp", + type=Path, + help="Existing temporary directory; overrides JOBLIB_TEMP_FOLDER", + ) + parser.add_argument("--max-workers", type=int, default=6) + parser.add_argument("--max-trials", type=int) + parser.add_argument("--verbose", action="store_true") + parser.add_argument("--use-rois", action="store_true") + parser.add_argument("--interactive", action="store_true") + parser.add_argument("--operator", default="SLAP2 User") + parser.add_argument("--analyze-hz", type=float, default=100.0) + parser.add_argument("--decay-tau-s", type=float, default=0.15) + parser.add_argument("--baseline-window-s", type=float, default=4.0) + parser.add_argument("--denoise-window-s", type=float, default=1.0) + parser.add_argument("--vif", type=float, default=1.38) + parser.add_argument("--d-xy", type=int, default=5) + parser.add_argument("--peak-buffer", type=int, default=3) + parser.add_argument("--psf-dilation", type=int, default=17) + return parser + + +def require_directory(path: Path) -> Path: + """Resolve a source directory and fail before staging if it is missing.""" + resolved = path.resolve(strict=True) + if not resolved.is_dir(): + raise NotADirectoryError(path) + return resolved + + +def validate_relative_names(values: object, field: str) -> None: + """Reject names that would bypass the explicitly selected input roots.""" + import numpy as np + + for entry in np.asarray(values, dtype=object).flat: + if entry is None or str(entry) == "": + continue + name = str(entry) + posix = PurePosixPath(name) + windows = PureWindowsPath(name) + if ( + posix.is_absolute() + or windows.anchor + or ".." in posix.parts + or ".." in windows.parts + ): + raise ValueError( + f"{field} must contain relative filenames without '..': " + f"{name!r}. Rebase the metadata explicitly before staging; " + "this example will not guess a replacement basename." + ) + + +def stage_directory(source: Path, destination: Path, mode: str) -> None: + """Copy or link a directory without replacing any existing destination.""" + if destination.exists() or destination.is_symlink(): + raise FileExistsError(destination) + if mode == "copy": + shutil.copytree(source, destination, symlinks=True) + elif mode == "symlink": + destination.symlink_to(source, target_is_directory=True) + else: + raise ValueError(f"Unknown staging mode: {mode}") + + +def validate_annotation_arguments( + args: argparse.Namespace, parser: argparse.ArgumentParser +) -> None: + """Reject inconsistent GUI policy before touching inputs.""" + if args.interactive and not args.use_rois: + parser.error("--interactive requires --use-rois") + if args.use_rois and not args.interactive and not args.annotations_dir: + parser.error("Headless --use-rois requires --annotations-dir") + + +def require_file(path: Path) -> Path: + """Resolve a required metadata file without creating anything.""" + resolved = path.resolve(strict=True) + if not resolved.is_file(): + raise FileNotFoundError(path) + return resolved + + +def validate_destination(requested: Path, sources: tuple[Path, ...]) -> Path: + """Refuse existing destinations and recursive staging into input trees.""" + if requested.exists() or requested.is_symlink(): + raise FileExistsError(f"Choose a fresh --results-dir: {requested}") + results = requested.resolve() + for source in sources: + if results.is_relative_to(source): + raise ValueError("Results must be outside all input directories") + return results + + +def validate_paths(args: argparse.Namespace) -> InputPaths: + """Check all paths without modifying files or the environment.""" + table = require_file(args.trial_table) + data = require_directory(args.data_dir) + motion = require_directory(args.motion_correction) + require_file(motion / "bandRegLookupTable.h5") + annotations = None + sources = (data, motion) + if args.annotations_dir is not None: + annotations = require_directory(args.annotations_dir) + require_file(annotations / "annotations.h5") + sources += (annotations,) + results = validate_destination(args.results_dir, sources) + joblib_temp = ( + require_directory(args.joblib_temp) + if args.joblib_temp is not None + else None + ) + return InputPaths(table, data, motion, results, annotations, joblib_temp) + + +def prepare_run(args: argparse.Namespace) -> PreparedRun: + """Validate metadata and options before staging or environment changes.""" + paths = validate_paths(args) + from giant_python import ( + AnnotationOptions, + BandSiloParams, + ExecutionOptions, + TrialTable, + ) + + table = TrialTable.from_h5(paths.table) + if table.slap2_info is None or not table.motion_correction: + raise ValueError("A pre-registered SLAP2 band trial table is required") + if "fn_adata" not in table.motion_correction: + raise ValueError("The table requires motion_correction/fn_adata") + validate_relative_names(table.filename, "filename") + validate_relative_names( + table.motion_correction["fn_adata"], "motion_correction/fn_adata" + ) + params = BandSiloParams( + analyze_hz=args.analyze_hz, + decay_tau_s=args.decay_tau_s, + baseline_window_s=args.baseline_window_s, + denoise_window_s=args.denoise_window_s, + vif=args.vif, + d_xy=args.d_xy, + peak_buffer=args.peak_buffer, + psf_dilation=args.psf_dilation, + ) + execution = ExecutionOptions( + max_workers=args.max_workers, + max_trials=args.max_trials, + verbose=args.verbose, + ) + annotations = AnnotationOptions( + enabled=args.use_rois, + interactive=args.interactive, + operator=args.operator, + ) + return PreparedRun(paths, table, params, execution, annotations) + + +@contextmanager +def joblib_environment( + directory: Path | None, +) -> Generator[None, None, None]: + """Temporarily override worker scratch space only for a validated run.""" + if directory is None: + yield + return + previous = os.environ.get("JOBLIB_TEMP_FOLDER") + os.environ["JOBLIB_TEMP_FOLDER"] = str(directory) + try: + yield + finally: + if previous is None: + os.environ.pop("JOBLIB_TEMP_FOLDER", None) + else: + os.environ["JOBLIB_TEMP_FOLDER"] = previous + + +def stage_and_extract(run: PreparedRun, args: argparse.Namespace) -> None: + """Stage validated inputs and pass the relocated model to extraction.""" + from giant_python import extract_band_sources + + paths = run.paths + results_dir = paths.results + data_dir = paths.data + + # No overwrite mode and no automatic cleanup: a failed run leaves its + # partial directory for inspection, never removes an existing tree/link. + results_dir.mkdir(parents=True, exist_ok=False) + staged_table = results_dir / "input_trial_table.h5" + shutil.copy2(paths.table, staged_table) + stage_directory( + paths.motion, results_dir / "motion_correction", args.stage_motion + ) + if args.stage_data != "none": + staged_data = results_dir / "input" + stage_directory(data_dir, staged_data, args.stage_data) + data_dir = staged_data + if paths.annotations is not None: + stage_directory(paths.annotations, results_dir / "annotations", "copy") + + # Copying the HDF5 file does not rewrite the paths stored inside it. + # Pass this overridden model, not staged_table, to make staging effective. + table = replace( + run.table, + datadr=data_dir, + savedr=results_dir, + source_path=staged_table, + ) + summary = extract_band_sources( + table, run.params, execution=run.execution, annotations=run.annotations + ) + print(f"Extracted {len(summary.paths)} imaging path(s)") + print(results_dir / "source_extraction" / "experiment_summary.h5") + + +def main(argv: list[str] | None = None) -> int: + """Validate first, then stage a fresh run and extract registered bands.""" + parser = build_parser() + args = parser.parse_args(argv) + validate_annotation_arguments(args, parser) + run = prepare_run(args) + with joblib_environment(run.paths.joblib_temp): + stage_and_extract(run, args) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml index 64464e8..c8cb56f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,6 +26,7 @@ dependencies = [ "h5py", "tifffile", "joblib", + "tqdm", "torch", "opencv-python", "pandas", @@ -33,9 +34,6 @@ dependencies = [ "slap2_utils @ git+https://github.com/m-xie/SLAP2_Utils.git@multidatafiles", ] -[project.optional-dependencies] -gui = ["napari"] - [project.scripts] giant = "giant_python.cli:main" diff --git a/src/giant_python/__init__.py b/src/giant_python/__init__.py index 48ba9ae..b9e94c7 100644 --- a/src/giant_python/__init__.py +++ b/src/giant_python/__init__.py @@ -1,45 +1,82 @@ -"""GIAnT-Python: Python translation of the GIAnT-MATLAB analysis package. - -Public API ----------- -The recommended entry point for the full pipeline is :class:`Pipeline`. Power -users can drive individual stages (``TrialTableBuilder``, ``MotionCorrector``, -``SourceExtractor``) or call the underlying functions directly. Typed data -models (``TrialTable``, ``AlignmentData``, ``ExperimentSummary``) own the -HDF5 serialization shared with GIAnT-MATLAB. - -The GUI lives in :mod:`giant_python.gui` and is imported lazily, so importing -this package never pulls in GUI dependencies. +"""GIAnT-Python: band extraction functions and a chainable ``Pipeline``. + +Use ``BandSiloParams`` for science, ``ExecutionOptions`` for run policy and +``AnnotationOptions`` for ROI/GUI policy. ``extract_sources`` defaults to the +implemented SLAP2 band workflow; explicitly requested unsupported modes raise. +Public exports are lazy: importing GIAnT does not load acquisition or GUI +toolkits. Summary arrays are currently eager, not disk-backed. """ -from .models import ( - AlignmentData, - AlignParams, - ExperimentSummary, - SiloParams, - TrialTable, -) -from .pipeline import ( - BandSourceExtractor, - MotionCorrector, - Pipeline, - SourceExtractor, - StandardSourceExtractor, - TrialTableBuilder, -) +from importlib import import_module +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .models import ( + AnnotationOptions, + BandSiloParams, + ExecutionOptions, + ExperimentSummary, + FrameInfo, + PathSummary, + TrialTable, + resolve_band_options, + ) + from .pipeline import ( + Pipeline, + annotate_rois, + extract_band_sources, + extract_sources, + ) __version__ = "0.1.10" +_MODEL_EXPORTS = { + "AnnotationOptions", + "BandSiloParams", + "ExecutionOptions", + "ExperimentSummary", + "FrameInfo", + "PathSummary", + "TrialTable", + "resolve_band_options", +} +_PIPELINE_EXPORTS = { + "Pipeline", + "annotate_rois", + "extract_band_sources", + "extract_sources", +} + + +def __getattr__(name: str): + """Load a public model or workflow export on first access.""" + if name in _MODEL_EXPORTS: + module = import_module(".models", __name__) + elif name in _PIPELINE_EXPORTS: + module = import_module(".pipeline", __name__) + else: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + value = getattr(module, name) + globals()[name] = value + return value + + +def __dir__() -> list[str]: + """Include lazy public exports in interactive completion.""" + return sorted(set(globals()) | set(__all__)) + + __all__ = [ - "AlignmentData", - "AlignParams", - "BandSourceExtractor", + "AnnotationOptions", + "BandSiloParams", + "ExecutionOptions", "ExperimentSummary", - "MotionCorrector", + "FrameInfo", + "PathSummary", "Pipeline", - "SiloParams", - "SourceExtractor", - "StandardSourceExtractor", "TrialTable", - "TrialTableBuilder", + "annotate_rois", + "extract_band_sources", + "extract_sources", + "resolve_band_options", ] diff --git a/src/giant_python/bandsilo/__init__.py b/src/giant_python/bandsilo/__init__.py deleted file mode 100644 index f0f6470..0000000 --- a/src/giant_python/bandsilo/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -"""Self-contained BandSILo (SLAP2 band-scan) backend. - -This package owns everything the band source-extraction backend needs -(HDF5 IO, geometry/PSF, trial-data reading, background/rho, peak detection, -NMF localization, trace extraction, and orchestration), ported faithfully from -``extractSLAP2IntegrationSources.py`` in ophys-slap2-analysis. - -It is deliberately self-contained: shared-looking kernels are duplicated here -rather than promoted to the generic ``math``/``io`` homes, so a future standard -(pixel-movie) SILo backend can be added without depending on this package. The -public entry points live in :mod:`giant_python.pipeline.extract`, which wraps -:mod:`giant_python.bandsilo.pipeline`. -""" diff --git a/src/giant_python/bandsilo/annotate.py b/src/giant_python/bandsilo/annotate.py deleted file mode 100644 index 1b988a4..0000000 --- a/src/giant_python/bandsilo/annotate.py +++ /dev/null @@ -1,340 +0,0 @@ -"""Standalone band-scan ROI annotation step (decoupled from extraction). - -The GIAnT-MATLAB workflow runs ROI annotation as a *separate* CodeOcean -capsule (``run_capsule`` + ``annotateROIs.m`` / -``loadSessionImagesForAnnotation``) that writes an ``annotations.h5`` the -extraction step later consumes. This -module is the BandSILo analog: :func:`annotate_band_rois` loads just enough of -a ``trial_table.h5`` to display the per-DMD reference images (no -background/rho/NMF compute), launches the ROI selector, and writes -``/annotations/annotations.h5``. - -The same load-or-draw seam (:func:`resolve_user_rois`) is reused by the -headless extraction driver, so annotation geometry is computed identically in -both places. Extraction never draws unless it resolves to an interactive -session (:func:`resolve_interactivity`); otherwise it fails fast with guidance. - -The interactive GUI and disk/``slap2_utils`` IO cannot run headless and are -excluded from coverage; the pure helpers (first-valid-trial, motion medians, -interactivity resolution, and the load/fail-fast branches of -:func:`resolve_user_rois`) are unit-tested. -""" - -from __future__ import annotations - -import os -import sys -from pathlib import Path -from typing import Optional, Union - -import numpy as np - -from ..models.params import SiloParams -from . import geometry as geo -from .gui import ( - compute_user_roi_geometry, - load_annotations_h5, - run_user_roi_selection, - save_annotations_h5, -) -from .hdf5 import compute_keep_trials, load_alignment_data_h5, load_trial_table -from .progress import log - - -def first_valid_trial(keep_trials: np.ndarray, dmd_ix: int) -> int: - """Return the first kept trial index for a DMD. - - Parameters - ---------- - keep_trials : ndarray of bool, shape (n_dmds, n_trials) - Per-DMD/-trial keep mask. - dmd_ix : int - 0-based DMD index. - - Returns - ------- - int - 0-based index of the first ``True`` entry in ``keep_trials[dmd_ix]``. - - Raises - ------ - ValueError - If no trial is kept for this DMD. - """ - row = np.asarray(keep_trials)[dmd_ix] - idxs = np.flatnonzero(row) - if idxs.size == 0: - raise ValueError(f"No valid trials for DMD {dmd_ix + 1}") - return int(idxs[0]) - - -def motion_median_from_adata(a_data: dict) -> tuple: - """Return the median ``(row, column, z)`` motion from alignment data. - - Mirrors the ``motion_medians`` computation in - ``extractSLAP2IntegrationSources.py``: the per-axis median of the rounded - downsampled motion estimates for one trial's alignment data. Both the - annotate step (drawing) and the extract step (loading) use this so the ROI - geometry is identical. - - Parameters - ---------- - a_data : dict - Alignment data from :func:`giant_python.bandsilo.hdf5\ -.load_alignment_data_h5` (``motionDSr``/``motionDSc``/``motionDSz``). - - Returns - ------- - tuple of int - ``(median_row, median_column, median_z)``. - """ - return ( - int(np.nanmedian(np.round(a_data["motionDSr"]))), - int(np.nanmedian(np.round(a_data["motionDSc"]))), - int(np.nanmedian(np.round(a_data["motionDSz"]))), - ) - - -def _stdin_is_tty() -> bool: # pragma: no cover - depends on runtime stdin - """Return True if stdin is attached to an interactive terminal.""" - return bool(getattr(sys.stdin, "isatty", lambda: False)()) - - -def resolve_interactivity(override: Optional[bool] = None) -> bool: - """Decide whether a drawing GUI may be launched in this process. - - Resolution order: an explicit ``override`` wins; otherwise the - ``GIANT_HEADLESS`` environment variable forces headless; otherwise fall - back to whether stdin is a TTY. A display probe (e.g. ``DISPLAY`` on Linux) - can be added here later without changing callers. - - Parameters - ---------- - override : bool or None - ``SiloParams.interactive`` (``True``/``False`` force the answer; - ``None`` auto-detects). - - Returns - ------- - bool - True if the process may open the ROI-drawing GUI. - """ - if override is not None: - return bool(override) - if os.environ.get("GIANT_HEADLESS"): - return False - return _stdin_is_tty() - - -def _empty_user_rois(n_dmds: int) -> dict: - """Return an all-empty per-DMD user-ROI selection for ``n_dmds`` paths.""" - keys = [f"DMD{d + 1}" for d in range(n_dmds)] - return { - "user_roi_masks": {k: [] for k in keys}, - "user_roi_superpixels": {k: [] for k in keys}, - "user_roi_labels": {k: [] for k in keys}, - "roi_records": {k: [] for k in keys}, - "annotated": False, - } - - -def resolve_user_rois( - annotation_dir: Union[str, Path], - n_dmds: int, - user_roi_geo: dict, - *, - interactive: bool, - ref_files: Optional[dict] = None, -) -> dict: - """Load ``annotations.h5`` if present, else draw (interactive) or fail. - - This is the shared seam between the standalone annotate step and the - headless extract step. When ``annotations.h5`` holds at least one ROI it is - loaded and returned. Otherwise, if ``interactive`` the drawing GUI runs and - the result is saved; if not, a :class:`RuntimeError` is raised telling the - user to run ``giant annotate`` first (so batch/cluster ``extract`` jobs - never block on a GUI). - - Parameters - ---------- - annotation_dir : str or Path - Directory holding (or to hold) ``annotations.h5``. - n_dmds : int - Number of DMD paths. - user_roi_geo : dict - Per-DMD geometry from :func:`compute_user_roi_geometry`. - interactive : bool - Whether a GUI may be launched (from :func:`resolve_interactivity`). - ref_files : dict, optional - ``{"DMD{N}": reference_file}`` provenance written into the file. - - Returns - ------- - dict - ``{"user_roi_masks", "user_roi_superpixels", "user_roi_labels", - "roi_records", "annotated"}``. - - Raises - ------ - RuntimeError - If drawing is required but the process is not interactive. - """ - path = os.path.join(str(annotation_dir), "annotations.h5") - skip, masks, superpixels, labels, records = load_annotations_h5( - path, n_dmds, user_roi_geo - ) - if skip: - return { - "user_roi_masks": masks, - "user_roi_superpixels": superpixels, - "user_roi_labels": labels, - "roi_records": records, - "annotated": True, - } - if not interactive: - raise RuntimeError( - "draw_user_rois is set but no usable ROIs were found at " - f"{path!r}, and this process is not interactive. Run " - "`giant annotate ` first, or run in an " - "interactive session (or pass interactive=True) to draw now." - ) - return _draw_and_save_user_rois( - annotation_dir, n_dmds, user_roi_geo, ref_files - ) - - -def _draw_and_save_user_rois( - annotation_dir: Union[str, Path], - n_dmds: int, - user_roi_geo: dict, - ref_files: Optional[dict], -) -> dict: # pragma: no cover - interactive GUI + disk IO - """Run the ROI selector, persist ``annotations.h5``, and return it.""" - os.makedirs(str(annotation_dir), exist_ok=True) - selection = run_user_roi_selection(user_roi_geo, n_dmds) - save_annotations_h5( - str(annotation_dir), - selection["roi_records"], - selection["user_roi_masks"], - n_dmds, - ref_files=ref_files, - ) - selection["annotated"] = True - return selection - - -def build_user_roi_geometry( - trial_table: dict, - lookup: dict, -) -> tuple: # pragma: no cover - reference-stack/aData disk IO - """Build per-DMD user-ROI geometry and reference files for a session. - - Loads each DMD's reference stack, superpixel index map, and - first-valid-trial motion medians, then derives the display geometry via - :func:`compute_user_roi_geometry`. Mirrors the ``soma_geo`` setup block of - ``main`` in ``extractSLAP2IntegrationSources.py``. - - Parameters - ---------- - trial_table : dict - Normalized trial table (provides ``datadr``, ``ref_stack``, - ``keep_trials``, ``fn_adata``, ``n_dmds``). - lookup : dict - Band-registration lookup table from - :func:`giant_python.bandsilo.geometry.load_lookup_table`. - - Returns - ------- - tuple - ``(user_roi_geo, ref_files)`` dicts keyed ``DMD{N}``. - """ - user_roi_geo, ref_files = {}, {} - datadr = str(trial_table["datadr"]) - keep_trials = trial_table["keep_trials"] - for dmd_ix in range(int(trial_table["n_dmds"])): - key = f"DMD{dmd_ix + 1}" - ref_stack, _, ref_file = geo.load_reference_stack( - datadr, trial_table["ref_stack"], dmd_ix - ) - smi = geo.build_subsample_matrix_inds( - lookup["allSuperPixelIDs"][key], lookup["sparseMaskInds"][key] - ) - fvt = first_valid_trial(keep_trials, dmd_ix) - a_data = load_alignment_data_h5(trial_table["fn_adata"][dmd_ix, fvt]) - user_roi_geo[key] = compute_user_roi_geometry( - ref_stack, - lookup["fastZ2RefZ"][key], - smi, - motion_median_from_adata(a_data), - ) - ref_files[key] = ref_file - return user_roi_geo, ref_files - - -def _resolve_params( - params_in, -) -> SiloParams: # pragma: no cover - trivial glue - """Coerce ``params_in`` (None / dict / SiloParams) to a SiloParams.""" - if params_in is None: - return SiloParams(scan_mode="band") - if isinstance(params_in, SiloParams): - return params_in - return SiloParams(**params_in) - - -def annotate_band_rois( - path_to_trial_table: Union[str, Path], - params_in=None, -) -> str: # pragma: no cover - full GUI + reference-stack/aData IO - """Annotate user ROIs for a band-scan session (standalone step). - - The BandSILo analog of the GIAnT-MATLAB annotation capsule: loads the - ``trial_table.h5`` and per-DMD reference geometry (no extraction compute), - then loads an existing ``annotations.h5`` or launches the ROI selector and - writes ``/annotations/annotations.h5``. Idempotent -- a session - that already has annotations short-circuits (matching the MATLAB - ``if exist(fnAnnH5)`` guard). - - Parameters - ---------- - path_to_trial_table : str or Path - Path to the ``trial_table.h5`` produced by BandRegistration. - params_in : SiloParams or dict, optional - Parameter overrides (``interactive`` gates the GUI). - - Returns - ------- - str - Path to the written (or existing) ``annotations.h5``. - """ - params = _resolve_params(params_in) - log(f"Loading trial table {path_to_trial_table}", params.verbose) - trial_table = load_trial_table(path_to_trial_table) - trial_table["keep_trials"] = compute_keep_trials( - trial_table["fn_adata"], - trial_table["filename"], - trial_table["datadr"], - ) - annotation_dir = trial_table["annotation_save_dr"] - - lookup = geo.load_lookup_table( - Path(trial_table["moco_save_dr"]) / "bandRegLookupTable.h5", - trial_table["n_dmds"], - ) - log( - f"Loading reference geometry for {trial_table['n_dmds']} " - "DMD path(s)", - params.verbose, - ) - user_roi_geo, ref_files = build_user_roi_geometry(trial_table, lookup) - - resolve_user_rois( - annotation_dir, - trial_table["n_dmds"], - user_roi_geo, - interactive=resolve_interactivity(params.interactive), - ref_files=ref_files, - ) - out_path = os.path.join(str(annotation_dir), "annotations.h5") - log(f"Annotations at {out_path}", params.verbose) - return out_path diff --git a/src/giant_python/bandsilo/background.py b/src/giant_python/bandsilo/background.py deleted file mode 100644 index e8deb53..0000000 --- a/src/giant_python/bandsilo/background.py +++ /dev/null @@ -1,1313 +0,0 @@ -"""Motion binning, background/noise estimation, and rho computation. - -Ported from the background/noise block of -``extractSLAP2IntegrationSources.py`` (ref 1901-2356). This module owns the -kernels that turn the per-superpixel low-res traces into: - -* motion bins (3-D and 2-D) and the selected-pixel support, -* the PSF convolution matrices ``D``/``D_expanded`` and the sparse per-motion - ``H`` matrices, -* the interpolated background estimate (``build_interp_data`` + - rolling-window baseline) and its reassembly onto the superpixel grid, -* an affine noise-variance model producing ``data_std`` and the z-scored - ``residual``, -* the ``rho`` matched-filter response and its NaN-aware temporal smoothing. - -The summary images that consume these outputs (the mean image, and the -activity image built from ``rho``) live in -:mod:`giant_python.bandsilo.summary_images`. The full threading of these -kernels over the per-DMD arrays is done by the Phase-8 pipeline driver; here -each kernel is a standalone, testable function. - -The commented-out soft-impute background completion and rolling-MAD rho -normalization in the reference are intentionally dropped (dead code). -""" - -from __future__ import annotations - -from typing import List, Tuple - -import numpy as np -import pandas as pd -import scipy.ndimage as ndimage -import torch -from scipy import signal -from scipy.interpolate import CubicSpline, RectBivariateSpline - -try: # pragma: no cover - exercised via the installed extra - import bottleneck as bn -except ImportError: # pragma: no cover - bn = None - -from .progress import progress - - -def bin_motion( - motion_r: np.ndarray, motion_c: np.ndarray, motion_z: np.ndarray -) -> Tuple[np.ndarray, np.ndarray]: - """Group frames into unique rounded (row, col, z) motion bins. - - Parameters - ---------- - motion_r, motion_c, motion_z : ndarray - Per-frame downsampled motion in rows, columns, and z. - - Returns - ------- - unique_motion : ndarray of shape (n_bins, 3) - The unique rounded ``[row, col, z]`` motion vectors. - mot_inds : ndarray of shape (n_frames,) - Index of each frame's motion bin into ``unique_motion``. - """ - unique_motion, mot_inds = np.unique( - np.round(np.stack((motion_r, motion_c, motion_z), axis=1)), - axis=0, - return_inverse=True, - ) - # np.unique's inverse is 1-D on numpy<2 and 2.1+, but 2-D on 2.0; flatten - # so downstream bincount/isin behave identically across versions. - return unique_motion, np.reshape(mot_inds, -1) - - -def select_motion_bins( - unique_motion: np.ndarray, - mot_inds: np.ndarray, - motion_z: np.ndarray, - z_thresh: float = 1.5, - min_frames: int = 100, -) -> Tuple[np.ndarray, np.ndarray]: - """Keep motion bins near the median z with enough frames. - - A bin is kept when its z is within ``z_thresh`` of the median z (over all - frames) and it contains more than ``min_frames`` frames. After filtering, - the remaining frames are treated as having no z motion. - - Parameters - ---------- - unique_motion : ndarray of shape (n_bins, 3) - Unique motion vectors from :func:`bin_motion`. - mot_inds : ndarray of shape (n_frames,) - Per-frame motion-bin indices. - motion_z : ndarray - Per-frame z motion (used for the median). - z_thresh : float - Maximum absolute z deviation from the median, in microns. - min_frames : int - Minimum frame count (strictly greater) for a bin to be kept. - - Returns - ------- - mot_inds_to_keep : ndarray - Indices of the kept bins into ``unique_motion``. - frames_to_keep : ndarray of bool - Per-frame mask of frames belonging to a kept bin. - """ - median_z = np.median(motion_z) - bin_counts = np.bincount(mot_inds, minlength=unique_motion.shape[0]) - keep_mask = (np.abs(unique_motion[:, 2] - median_z) <= z_thresh) & ( - bin_counts > min_frames - ) - mot_inds_to_keep = np.nonzero(keep_mask)[0] - frames_to_keep = np.isin(mot_inds, mot_inds_to_keep) - return mot_inds_to_keep, frames_to_keep - - -def bin_motion_yx( - motion_r: np.ndarray, - motion_c: np.ndarray, - frames_to_keep: np.ndarray, -) -> Tuple[np.ndarray, np.ndarray]: - """Bin kept frames into unique rounded (row, col) motion bins. - - Parameters - ---------- - motion_r, motion_c : ndarray - Per-frame downsampled row/column motion. - frames_to_keep : ndarray of bool - Frames to bin; dropped frames get index ``-1``. - - Returns - ------- - unique_motion_to_keep_yx : ndarray of shape (n_yx_bins, 2) - Unique rounded ``[row, col]`` motion vectors over the kept frames. - mot_inds_yx : ndarray of shape (n_frames,), int32 - Per-frame bin index into ``unique_motion_to_keep_yx``; ``-1`` for - frames not kept. - """ - mot_inds_yx = -1 * np.ones((frames_to_keep.shape[0],), dtype=np.int32) - stacked = np.round( - np.stack((motion_r, motion_c), axis=1)[frames_to_keep, :] - ) - unique_motion_to_keep_yx, inv = np.unique( - stacked, axis=0, return_inverse=True - ) - mot_inds_yx[frames_to_keep] = np.reshape(inv, -1) - return unique_motion_to_keep_yx, mot_inds_yx - - -def build_selected_pixel_mask( - unique_motion_to_keep_yx: np.ndarray, - ref_d: np.ndarray, - ref_r: np.ndarray, - ref_c: np.ndarray, - num_fast_zs: int, - dmd_pixels_per_column: int, - dmd_pixels_per_row: int, - psf2d: np.ndarray, -) -> Tuple[np.ndarray, np.ndarray]: - """Build the selected-pixel support mask over all kept motion shifts. - - Marks every reference pixel shifted by each kept 2-D motion bin, then - dilates in-plane by the PSF footprint. - - Parameters - ---------- - unique_motion_to_keep_yx : ndarray of shape (n_yx_bins, 2) - Kept 2-D motion vectors. - ref_d, ref_r, ref_c : ndarray - Per-superpixel reference depth/row/column indices. - num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row : int - Grid geometry. - psf2d : ndarray - The (cropped) PSF for this DMD; its shape sets the dilation footprint. - - Returns - ------- - sel_pix_mask : ndarray of bool - ``(num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row)`` support. - sel_pix_idxs : ndarray of int - Flattened indices of the selected pixels. - """ - sel_pix_mask = np.zeros( - (num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row), dtype=bool - ) - for i in range(len(unique_motion_to_keep_yx)): - sel_pix_mask[ - ref_d, - ref_r + int(unique_motion_to_keep_yx[i, 0]), - ref_c + int(unique_motion_to_keep_yx[i, 1]), - ] = True - sel_pix_mask = ndimage.binary_dilation( - sel_pix_mask, - structure=np.ones((1, psf2d.shape[0], psf2d.shape[1]), dtype=bool), - ) - sel_pix_idxs = np.flatnonzero(sel_pix_mask) - return sel_pix_mask, sel_pix_idxs - - -def pixel_coords_from_idxs( - sel_pix_idxs: np.ndarray, - dmd_pixels_per_column: int, - dmd_pixels_per_row: int, -) -> np.ndarray: - """Convert flat selected-pixel indices to ``[z, row, col]`` coordinates. - - Parameters - ---------- - sel_pix_idxs : ndarray of int - Flattened selected-pixel indices. - dmd_pixels_per_column, dmd_pixels_per_row : int - Grid geometry. - - Returns - ------- - ndarray of shape (n_sel, 3), int32 - ``[z, row, col]`` per selected pixel. - """ - plane_size = dmd_pixels_per_column * dmd_pixels_per_row - pixel_coords = np.empty((len(sel_pix_idxs), 3), dtype=np.int32) - pixel_coords[:, 0] = sel_pix_idxs // plane_size - remainder = sel_pix_idxs % plane_size - pixel_coords[:, 1] = remainder // dmd_pixels_per_row - pixel_coords[:, 2] = remainder % dmd_pixels_per_row - return pixel_coords - - -def expand_psf( - psf2d: np.ndarray, ex_fac: int = 2 -) -> Tuple[torch.Tensor, Tuple[int, int], torch.Tensor, Tuple[int, int]]: - """Normalize a PSF and build a center-aligned higher-resolution copy. - - The expanded PSF is an ``ex_fac``x up-sampled cubic-spline interpolation - that keeps the center aligned, used to form the ``HD - HD_expanded`` - matched filter in :func:`compute_rho`. - - Parameters - ---------- - psf2d : ndarray - The (cropped) PSF for this DMD. - ex_fac : int - Expansion factor along each axis. - - Returns - ------- - psf_tensor : torch.Tensor - Sum-normalized PSF. - psf_center : tuple of int - ``(row, col)`` center of ``psf_tensor``. - psf_tensor_expanded : torch.Tensor - Sum-normalized expanded PSF. - psf_center_expanded : tuple of int - ``(row, col)`` center of ``psf_tensor_expanded``. - """ - psf_arr = np.asarray(psf2d, dtype=np.float32) - psf_tensor = torch.from_numpy(psf_arr).float() - psf_tensor = psf_tensor / torch.sum(psf_tensor) - psf_shape = psf_arr.shape - psf_center = (psf_shape[0] // 2, psf_shape[1] // 2) - - expanded_center_y = (psf_shape[0] * ex_fac - 1) / 2 - expanded_center_x = (psf_shape[1] * ex_fac - 1) / 2 - orig_y = torch.linspace( - -expanded_center_y, expanded_center_y, psf_shape[0] - ) - orig_x = torch.linspace( - -expanded_center_x, expanded_center_x, psf_shape[1] - ) - expanded_y = torch.linspace( - -expanded_center_y, expanded_center_y, psf_shape[0] * ex_fac - ) - expanded_x = torch.linspace( - -expanded_center_x, expanded_center_x, psf_shape[1] * ex_fac - ) - interp_spline = RectBivariateSpline( - orig_y.numpy(), orig_x.numpy(), psf_tensor.numpy() - ) - psf_tensor_expanded = torch.from_numpy( - interp_spline(expanded_y.numpy(), expanded_x.numpy()) - ).float() - psf_tensor_expanded = psf_tensor_expanded / torch.sum(psf_tensor_expanded) - psf_shape_expanded = psf_tensor_expanded.shape - psf_center_expanded = ( - psf_shape_expanded[0] // 2, - psf_shape_expanded[1] // 2, - ) - return psf_tensor, psf_center, psf_tensor_expanded, psf_center_expanded - - -def shrink_psf( - psf2d: np.ndarray, scale_y: float = 1.0, scale_x: float = 0.75 -) -> np.ndarray: - """Spatially shrink a PSF on the same grid (TEMPORARY). - - Resamples the PSF on a grid whose coordinates are divided by the per-axis - scale so a ``scale < 1`` narrows the PSF's spatial footprint along that - axis while keeping the same array shape and center; the result is - re-normalized to sum to 1. Used only for the temporary activity-image - experiment. - - Parameters - ---------- - psf2d : ndarray - The (cropped) PSF for this DMD. - scale_y : float - Spatial shrink factor along the row (y) axis (``< 1`` shrinks, - ``1`` is a no-op). - scale_x : float - Spatial shrink factor along the column (x) axis (``< 1`` shrinks, - ``1`` is a no-op). - - Returns - ------- - ndarray of shape ``psf2d.shape``, float32 - The shrunk, sum-normalized PSF. - """ - psf_arr = np.asarray(psf2d, dtype=np.float32) - psf_shape = psf_arr.shape - center_y = (psf_shape[0] - 1) / 2 - center_x = (psf_shape[1] - 1) / 2 - orig_y = np.arange(psf_shape[0], dtype=np.float64) - center_y - orig_x = np.arange(psf_shape[1], dtype=np.float64) - center_x - interp_spline = RectBivariateSpline(orig_y, orig_x, psf_arr) - shrunk = interp_spline(orig_y / scale_y, orig_x / scale_x) - shrunk = np.clip(shrunk, 0.0, None).astype(np.float32) - total = shrunk.sum() - if total > 0: - shrunk = shrunk / total - return shrunk - - -def gaussian_kernel_2d( - sigma: float, truncate: float = 4.0 -) -> Tuple[torch.Tensor, Tuple[int, int]]: - """Build a sum-normalized isotropic 2-D Gaussian kernel (TEMPORARY). - - Used by the temporary difference-of-Gaussians activity-image experiment, - which replaces the PSF-derived ``D``/``D_expanded`` convolution matrices - with an isotropic center/surround pair. The subtraction itself still - happens in :func:`compute_rho`, after per-column normalization. - - Parameters - ---------- - sigma : float - Gaussian standard deviation, in pixels. - truncate : float - Kernel half-width, in standard deviations. Truncation lowers the - realized standard deviation slightly, so the default is generous. - - Returns - ------- - kernel : torch.Tensor - Sum-normalized ``(2 * radius + 1, 2 * radius + 1)`` kernel. - center : tuple of int - ``(row, col)`` center index of ``kernel``. - """ - radius = int(np.ceil(truncate * sigma)) - coords = np.arange(-radius, radius + 1, dtype=np.float64) - g_1d = np.exp(-(coords**2) / (2.0 * sigma**2)) - kernel = np.outer(g_1d, g_1d).astype(np.float32) - kernel /= kernel.sum() - return torch.from_numpy(kernel), (radius, radius) - - -def selected_pixels_2d_for_plane( - sel_pix_idxs: np.ndarray, - plane_z: int, - dmd_pixels_per_column: int, - dmd_pixels_per_row: int, -) -> Tuple[np.ndarray, np.ndarray]: - """Return the in-plane ``[row, col]`` coords of one z-plane's pixels. - - Parameters - ---------- - sel_pix_idxs : ndarray of int - Flattened selected-pixel indices (all planes). - plane_z : int - The z-plane to select. - dmd_pixels_per_column, dmd_pixels_per_row : int - Grid geometry. - - Returns - ------- - z_idxs : ndarray - Positions (into ``sel_pix_idxs``) of pixels in ``plane_z``. - sel_pixels_2d : ndarray of shape (n_plane, 2) - ``[row, col]`` coordinates of those pixels. - """ - plane_size = dmd_pixels_per_column * dmd_pixels_per_row - sel_pix_z = sel_pix_idxs // plane_size - sel_pix_remainder = sel_pix_idxs % plane_size - z_idxs = np.flatnonzero(sel_pix_z == plane_z) - sel_pixels_2d = np.column_stack( - [ - sel_pix_remainder[z_idxs] // dmd_pixels_per_row, - sel_pix_remainder[z_idxs] % dmd_pixels_per_row, - ] - ) - return z_idxs, sel_pixels_2d - - -def build_convolution_matrix( - sel_pixels_2d: np.ndarray, - psf_tensor: torch.Tensor, - psf_center: Tuple[int, int], -) -> torch.Tensor: - """Build a dense PSF-convolution matrix for one plane's selected pixels. - - Entry ``(t, s)`` is the PSF weight for the offset from source pixel ``s`` - to target pixel ``t`` (zero where the offset falls outside the PSF). Used - with both the base and expanded PSFs to form ``D``/``D_expanded``. - - Parameters - ---------- - sel_pixels_2d : ndarray of shape (n_plane, 2) - In-plane ``[row, col]`` coordinates. - psf_tensor : torch.Tensor - Sum-normalized PSF (base or expanded). - psf_center : tuple of int - ``(row, col)`` center of ``psf_tensor``. - - Returns - ------- - torch.Tensor - Dense ``(n_plane, n_plane)`` convolution matrix. - """ - n = sel_pixels_2d.shape[0] - if n == 0: - return torch.zeros((0, 0), dtype=torch.float32) - src_rows = sel_pixels_2d[:, 0][np.newaxis, :] - src_cols = sel_pixels_2d[:, 1][np.newaxis, :] - tgt_rows = sel_pixels_2d[:, 0][:, np.newaxis] - tgt_cols = sel_pixels_2d[:, 1][:, np.newaxis] - rel_rows = tgt_rows - src_rows + psf_center[0] - rel_cols = tgt_cols - src_cols + psf_center[1] - psf_np = psf_tensor.numpy() - psf_shape = psf_np.shape - valid_mask = ( - (rel_rows >= 0) - & (rel_rows < psf_shape[0]) - & (rel_cols >= 0) - & (rel_cols < psf_shape[1]) - ) - out = np.zeros((n, n), dtype=np.float32) - out[valid_mask] = psf_np[rel_rows[valid_mask], rel_cols[valid_mask]] - return torch.from_numpy(out) - - -def _collect_interp_columns( - cols, s_c, s_r, sel_idxs_by_col_all, sel_rows, r0, r1 -): - """Group a motion bin's reference pixels by column for interpolation. - - Columns with a single reference pixel become ``single_points`` (copied - verbatim); columns with several become sorted ``(rows, source-index)`` - entries for linear interpolation. - """ - rows_by_col = {} - idxs_by_col = {} - single_points = [] - for c in cols: - c_int = int(c) - if c_int not in sel_idxs_by_col_all: - continue - data_ix = np.flatnonzero(s_c == c) - if data_ix.size == 1: - rr = int(s_r[data_ix[0]]) - col_sel = sel_idxs_by_col_all[c_int] - rr_match = col_sel[sel_rows[col_sel] == rr] - if rr_match.size > 0 and (r0 <= rr <= r1): - single_points.append((rr_match, int(data_ix[0]))) - continue - rows = s_r[data_ix] - order = rows.argsort(kind="mergesort") - rows_by_col[c_int] = rows[order] - idxs_by_col[c_int] = data_ix[order] - return rows_by_col, idxs_by_col, single_points - - -def _interp_columns( - out, - rows_by_col, - idxs_by_col, - sel_idxs_by_col_all, - sel_rows, - data, - frames, - dtype, - method="linear", - spline_weight_cache=None, -): - """Interpolate each multi-point column onto its selected pixels. - - Selected rows that coincide with a reference row are copied exactly; those - strictly between reference rows use linear or natural-cubic interpolation; - those outside the sampled row range are left untouched (NaN). Cubic spline - weights depend only on row geometry, so they are computed once and applied - to all complete frames in one matrix multiplication. Linear interpolation - is retained for columns with fewer than three samples and frames containing - missing source values. - """ - if method not in {"linear", "cubic"}: - raise ValueError( - "interpolation method must be 'linear' or 'cubic', " - f"got {method!r}" - ) - for c_int, rows in rows_by_col.items(): - col_sel_pix_idxs = sel_idxs_by_col_all[c_int] - target_rows = sel_rows[col_sel_pix_idxs] - src_idx = idxs_by_col[c_int] - - pos = np.searchsorted(rows, target_rows, side="left") - in_bounds = pos < rows.size - exact = np.zeros_like(pos, dtype=bool) - exact[in_bounds] = rows[pos[in_bounds]] == target_rows[in_bounds] - - if np.any(exact): - exact_rows = np.flatnonzero(exact) - exact_src = src_idx[pos[exact]] - out[np.ix_(col_sel_pix_idxs[exact_rows], frames)] = data[ - exact_src - ][:, frames] - - lo = pos - 1 - hi = pos - interp_mask = (~exact) & (lo >= 0) & (hi < rows.size) - if not np.any(interp_mask): - continue - - interp_rows = np.flatnonzero(interp_mask) - lo_v = lo[interp_mask] - hi_v = hi[interp_mask] - row_lo = rows[lo_v].astype(dtype, copy=False) - row_hi = rows[hi_v].astype(dtype, copy=False) - denom = row_hi - row_lo - nonzero = denom != 0 - if not np.any(nonzero): - # Unreachable defensive guard (mirrors the reference): sorted - # distinct rows make searchsorted never straddle equal rows, so - # denom is always non-zero here. - continue # pragma: no cover - - interp_rows = interp_rows[nonzero] - lo_v = lo_v[nonzero] - hi_v = hi_v[nonzero] - row_lo = row_lo[nonzero] - row_hi = row_hi[nonzero] - alpha = ( - (target_rows[interp_rows].astype(dtype) - row_lo) - / (row_hi - row_lo) - )[:, None] - - target_indices = col_sel_pix_idxs[interp_rows] - linear_frame_mask = np.ones(frames.size, dtype=bool) - has_cubic_geometry = rows.size >= 3 and np.all(np.diff(rows) > 0) - if method == "cubic" and has_cubic_geometry: - source_values = data[src_idx][:, frames] - complete_frames = np.all(np.isfinite(source_values), axis=0) - if np.any(complete_frames): - source_rows = rows.astype(dtype, copy=False) - target_interp_rows = target_rows[interp_rows].astype( - dtype, copy=False - ) - origin = source_rows[0] - cache_key = ( - tuple(source_rows - origin), - tuple(target_interp_rows - origin), - np.dtype(dtype).str, - ) - spline_weights = ( - spline_weight_cache.get(cache_key) - if spline_weight_cache is not None - else None - ) - if spline_weights is None: - spline_weights = CubicSpline( - source_rows, - np.eye(rows.size, dtype=dtype), - axis=0, - bc_type="natural", - extrapolate=False, - )(target_interp_rows).astype(dtype, copy=False) - if spline_weight_cache is not None: - spline_weight_cache[cache_key] = spline_weights - cubic_values = ( - spline_weights @ source_values[:, complete_frames] - ) - out[ - np.ix_(target_indices, frames[complete_frames]) - ] = cubic_values - linear_frame_mask = ~complete_frames - - if not np.any(linear_frame_mask): - continue - linear_frames = frames[linear_frame_mask] - vals_lo = data[src_idx[lo_v]][:, linear_frames] - vals_hi = data[src_idx[hi_v]][:, linear_frames] - linear_values = (1.0 - alpha) * vals_lo + alpha * vals_hi - out[np.ix_(target_indices, linear_frames)] = linear_values - - -def build_interp_data( - data: np.ndarray, - ref_r: np.ndarray, - ref_c: np.ndarray, - sel_pixels_2d: np.ndarray, - unique_motion_to_keep_yx: np.ndarray, - mot_inds_yx: np.ndarray, - height: int = 800, - width: int = 1280, - dtype=np.float32, - method: str = "cubic", -) -> Tuple[np.ndarray, Tuple[int, int], Tuple[int, int]]: - """Interpolate reference-pixel traces onto the selected-pixel grid. - - For each kept motion bin, the reference pixels (shifted by the bin's - motion) are linearly interpolated along rows, per column, onto the selected - pixels of the plane, filling one background estimate per selected pixel and - frame. Selected pixels outside the sampled row range remain NaN. - - Parameters - ---------- - data : ndarray of shape (n_ref, n_frames) - Reference-pixel traces for one plane. - ref_r, ref_c : ndarray - Reference-pixel row/column indices for this plane. - sel_pixels_2d : ndarray of shape (n_sel, 2) - Selected-pixel ``[row, col]`` coordinates for this plane. - unique_motion_to_keep_yx : ndarray of shape (n_yx_bins, 2) - Kept 2-D motion vectors. - mot_inds_yx : ndarray of shape (n_frames,) - Per-frame motion-bin index (``-1`` for dropped frames). - height, width : int - Sensor row/column extents used to window valid shifts. - dtype : numpy dtype - Working/output dtype. - method : {"linear", "cubic"} - Interpolation along each column (default ``"cubic"``). Cubic mode uses - a natural spline with precomputed geometry weights and falls back to - linear for columns with fewer than three samples or frames with missing - samples. - - Returns - ------- - out : ndarray of shape (n_sel, n_frames) - Interpolated background estimate (NaN where unsampled). - row_bounds : tuple of int - ``(r0, r1)`` inclusive row window. - col_bounds : tuple of int - ``(c0, c1)`` inclusive column window. - """ - ref_r = np.asarray(ref_r, dtype=np.int32) - ref_c = np.asarray(ref_c, dtype=np.int32) - data = np.asarray(data, dtype=dtype) - if method not in {"linear", "cubic"}: - raise ValueError( - "interpolation method must be 'linear' or 'cubic', " - f"got {method!r}" - ) - - r0 = int(max(0, ref_r.min()) + unique_motion_to_keep_yx[:, 0].min()) - r1 = int(min(height, ref_r.max()) + unique_motion_to_keep_yx[:, 0].max()) - c0 = int(max(0, ref_c.min()) + unique_motion_to_keep_yx[:, 1].min()) - c1 = int(min(width, ref_c.max()) + unique_motion_to_keep_yx[:, 1].max()) - - out = np.full((sel_pixels_2d.shape[0], data.shape[1]), np.nan, dtype=dtype) - - sel_cols = sel_pixels_2d[:, 1] - sel_rows = sel_pixels_2d[:, 0] - unique_sel_cols = np.unique(sel_cols) - sel_idxs_by_col_all = { - int(c): np.flatnonzero(sel_cols == c) for c in unique_sel_cols - } - - frames_by_motion = [ - np.flatnonzero(mot_inds_yx == idx) - for idx in range(len(unique_motion_to_keep_yx)) - ] - spline_weight_cache = {} if method == "cubic" else None - - for m_idx, frames in enumerate(frames_by_motion): - if frames.size == 0: - continue - - s_r = ref_r + int(unique_motion_to_keep_yx[m_idx, 0]) - s_c = ref_c + int(unique_motion_to_keep_yx[m_idx, 1]) - - in_win = (s_r >= r0) & (s_r <= r1) & (s_c >= c0) & (s_c <= c1) - if not np.any(in_win): - continue - cols = np.unique(s_c[in_win]) - - rows_by_col, idxs_by_col, single_points = _collect_interp_columns( - cols, s_c, s_r, sel_idxs_by_col_all, sel_rows, r0, r1 - ) - _interp_columns( - out, - rows_by_col, - idxs_by_col, - sel_idxs_by_col_all, - sel_rows, - data, - frames, - dtype, - method, - spline_weight_cache, - ) - for rr_match, src_pos in single_points: - out[np.ix_(rr_match, frames)] = data[src_pos, frames] - - return out, (r0, r1), (c0, c1) - - -def baseline_window_frames(align_hz: float, baseline_window_s: float) -> int: - """Return the rolling-baseline window length in frames. - - Parameters - ---------- - align_hz : float - Alignment/analysis rate, in Hz. - baseline_window_s : float - Baseline window, in seconds. - - Returns - ------- - int - ``int(align_hz * baseline_window_s)``. - """ - return int(align_hz * baseline_window_s) - - -def compute_rolling_baseline( - interp_data: np.ndarray, baseline_window: int -) -> np.ndarray: - """Compute a NaN-aware rolling-median baseline of the interpolated data. - - Each frame's baseline is the median of its centered window, ignoring NaNs, - so windows with no valid samples stay NaN and edge windows are truncated to - the available samples. This is the fast moving-median analogue of the - former moving-mean baseline; the median is more robust to transient - activity spikes when estimating the background. - - The heavy lifting uses :func:`bottleneck.move_median` (a C sliding-window - median whose cost is close to the moving mean) when available, falling back - to pandas' skip-list rolling median otherwise. ``bottleneck`` computes a - *trailing* window, so the data is right-padded with NaNs by ``window // 2`` - and the result is sliced back to center each window on its frame. - - Parameters - ---------- - interp_data : ndarray of shape (n_sel, n_frames) - Interpolated background estimate (from :func:`build_interp_data`). - baseline_window : int - Rolling window length, in frames. - - Returns - ------- - ndarray - The rolling-median background estimate (NaN where no valid samples). - """ - n_frames = interp_data.shape[1] - window = int(min(max(baseline_window, 1), n_frames)) - - if bn is not None: - pad = window // 2 - if pad: - padded = np.concatenate( - [ - interp_data, - np.full( - (interp_data.shape[0], pad), - np.nan, - dtype=interp_data.dtype, - ), - ], - axis=1, - ) - else: - padded = interp_data - trailing = bn.move_median( - padded, window=window, min_count=1, axis=1 - ) - background = trailing[:, pad : pad + n_frames] - return background.astype(interp_data.dtype, copy=False) - - # Fallback (no bottleneck): pandas rolls along the row axis, so transpose - # to (n_frames, n_sel), roll each column, and transpose back. - background = ( - pd.DataFrame(interp_data.T) - .rolling(window=window, center=True, min_periods=1) - .median() - .to_numpy() - .T - ) - return background.astype(interp_data.dtype, copy=False) - - -def assemble_background( - interp_data_background: np.ndarray, - unique_motion_to_keep_yx: np.ndarray, - mot_inds_yx: np.ndarray, - sel_pix_idxs: np.ndarray, - ref_d: np.ndarray, - ref_r: np.ndarray, - ref_c: np.ndarray, - n_super_pixels: int, - n_frames: int, - dmd_pixels_per_column: int, - dmd_pixels_per_row: int, -) -> np.ndarray: - """Map the selected-pixel background back onto superpixels per motion bin. - - For each kept motion bin and its frames, each superpixel's motion-shifted - reference pixel is looked up in ``sel_pix_idxs`` and its background trace - is copied onto that superpixel. - - Parameters - ---------- - interp_data_background : ndarray of shape (n_sel, n_frames) - Rolling-baseline background on the selected-pixel grid. - unique_motion_to_keep_yx : ndarray of shape (n_yx_bins, 2) - Kept 2-D motion vectors. - mot_inds_yx : ndarray of shape (n_frames,) - Per-frame motion-bin index (``-1`` for dropped frames). - sel_pix_idxs : ndarray of int - Sorted flat selected-pixel indices. - ref_d, ref_r, ref_c : ndarray - Per-superpixel reference depth/row/column indices. - n_super_pixels, n_frames : int - Output shape. - dmd_pixels_per_column, dmd_pixels_per_row : int - Grid geometry. - - Returns - ------- - ndarray of shape (n_super_pixels, n_frames), float32 - Per-superpixel background (NaN where unmapped). - """ - plane_size = dmd_pixels_per_column * dmd_pixels_per_row - background = np.full((n_super_pixels, n_frames), np.nan, dtype=np.float32) - - for motion_idx in range(len(unique_motion_to_keep_yx)): - motion_frames = np.flatnonzero(mot_inds_yx == motion_idx) - d_r = int(unique_motion_to_keep_yx[motion_idx, 0]) - d_c = int(unique_motion_to_keep_yx[motion_idx, 1]) - s_d = ref_d - s_r = ref_r + d_r - s_c = ref_c + d_c - - shifted_indices = s_d * plane_size + s_r * dmd_pixels_per_row + s_c - - bg_idxs = np.searchsorted(sel_pix_idxs, shifted_indices) - idxs_mask = bg_idxs < sel_pix_idxs.size - idxs_mask[idxs_mask] &= ( - sel_pix_idxs[bg_idxs[idxs_mask]] == shifted_indices[idxs_mask] - ) - - if np.any(idxs_mask): - valid_rows = np.flatnonzero(idxs_mask) - background[ - np.ix_(valid_rows, motion_frames) - ] = interp_data_background[ - np.ix_(bg_idxs[valid_rows], motion_frames) - ] - - return background - - -def fit_noise_variance_model( - low_res_data_norm: np.ndarray, - background: np.ndarray, - v_im: np.ndarray, - vif: float = 1.38, -) -> Tuple[np.ndarray, float, float]: - """Calibrate an affine noise-variance model and return per-sample std. - - Fits ``Var ~= Vk * (background * vIM) + Vb`` from the brightest pixels over - the first valid frames, then returns the predicted standard deviation for - every sample. - - Parameters - ---------- - low_res_data_norm : ndarray of shape (n_superpixels, n_frames) - Count-normalized channel-1 traces. - background : ndarray of shape (n_superpixels, n_frames) - Per-superpixel background estimate. - v_im : ndarray of shape (n_superpixels, n_frames) - Inverse count image (``1 / lowResDataCt``). - vif : float - Variance inflation factor. - - Returns - ------- - data_std : ndarray - Per-sample predicted standard deviation. - v_k : float - Fitted multiplicative variance coefficient. - v_b : float - Fitted additive variance floor. - """ - first_valid_frames = np.flatnonzero( - np.any(~np.isnan(background[:, :1000]), axis=0) - ) - var_im = np.nanvar(low_res_data_norm[:, first_valid_frames], axis=1) - v_b = np.nanpercentile(var_im, 5) * vif - var_pred = np.nanmean( - background[:, first_valid_frames], axis=1 - ) * np.nanmean(v_im[:, first_valid_frames], axis=1) - sel_bright = var_pred > np.nanpercentile(var_pred, 90) - v_k = np.nanpercentile( - (var_im[sel_bright] - (v_b / vif)) / var_pred[sel_bright], 10 - ) - data_std = np.sqrt(np.clip(v_k * background * v_im, 0, None) + v_b) - return data_std, v_k, v_b - - -def compute_residual( - low_res_data_norm: np.ndarray, - background: np.ndarray, - data_std: np.ndarray, -) -> np.ndarray: - """Return the background-subtracted, noise-normalized residual. - - Parameters - ---------- - low_res_data_norm : ndarray - Count-normalized traces. - background : ndarray - Per-superpixel background estimate. - data_std : ndarray - Per-sample predicted standard deviation. - - Returns - ------- - ndarray - ``(low_res_data_norm - background) / data_std``. - """ - return (low_res_data_norm - background) / data_std - - -def build_motion_h_matrices( - sparse_h_inds: np.ndarray, - sparse_h_vals: np.ndarray, - unique_motion_to_keep_yx: np.ndarray, - sel_pix_idxs: np.ndarray, - num_super_pixels: int, - dmd_pixels_per_row: int, -) -> List[torch.Tensor]: - """Build one sparse superpixel<-selected-pixel ``H`` per motion bin. - - Each motion bin shifts the base ``H`` columns (image pixels) by its 2-D - motion, then remaps them into selected-pixel column space. - - Parameters - ---------- - sparse_h_inds : ndarray of shape (2, nnz) - Base ``H`` COO row/column indices (from ``geometry.build_sparse_h``). - sparse_h_vals : ndarray of shape (nnz,) - Base ``H`` values. - unique_motion_to_keep_yx : ndarray of shape (n_yx_bins, 2) - Kept 2-D motion vectors. - sel_pix_idxs : ndarray of int - Sorted flat selected-pixel indices. - num_super_pixels : int - Number of superpixels (``H`` row count). - dmd_pixels_per_row : int - Grid geometry (column-shift stride). - - Returns - ------- - list of torch.Tensor - One sparse COO ``(num_super_pixels, n_sel)`` tensor per motion bin. - """ - n_mot = len(unique_motion_to_keep_yx) - h_mots: List[torch.Tensor] = [None] * n_mot - base_sparse_rows = sparse_h_inds[0] - base_sparse_cols = sparse_h_inds[1] - motion_shifts = unique_motion_to_keep_yx[:, 0].astype( - np.int64, copy=False - ) * dmd_pixels_per_row + unique_motion_to_keep_yx[:, 1].astype( - np.int64, copy=False - ) - shifted_inds = np.empty((2, base_sparse_cols.shape[0]), dtype=np.int64) - shifted_inds[0] = base_sparse_rows - for i, pix_shift in enumerate(motion_shifts): - shifted_cols = base_sparse_cols + pix_shift - shifted_inds[1] = np.searchsorted(sel_pix_idxs, shifted_cols) - h_mots[i] = torch.sparse_coo_tensor( - shifted_inds, - sparse_h_vals, - (num_super_pixels, sel_pix_idxs.shape[0]), - dtype=torch.float32, - ) - return h_mots - - -def _plane_column_maps( - sel_pix_idxs: np.ndarray, num_fast_zs: int, plane_size: int -) -> Tuple[list, list, list]: - """Precompute per-z column masks, indices, and remaps into local space. - - Returns three lists (one entry per z): the boolean column mask, the column - indices of that plane's selected pixels, and a remap from global to local - (per-plane) column index (``-1`` outside the plane). - """ - sel_pix_z = sel_pix_idxs // plane_size - z_masks = [torch.from_numpy(sel_pix_z == z) for z in range(num_fast_zs)] - z_col_idxs = [torch.nonzero(m, as_tuple=False).squeeze(1) for m in z_masks] - z_remaps = [] - for z in range(num_fast_zs): - remap = torch.full((sel_pix_idxs.shape[0],), -1, dtype=torch.long) - if z_col_idxs[z].numel() > 0: - remap[z_col_idxs[z]] = torch.arange( - z_col_idxs[z].numel(), dtype=torch.long - ) - z_remaps.append(remap) - return z_masks, z_col_idxs, z_remaps - - -def _valid_sel_cols_for_motion( - motion_idx: int, - unique_motion_to_keep_yx: np.ndarray, - ref_d: np.ndarray, - ref_r: np.ndarray, - ref_c: np.ndarray, - sel_pix_idxs: np.ndarray, - num_fast_zs: int, - dmd_pixels_per_column: int, - dmd_pixels_per_row: int, - psf2d: np.ndarray, -) -> np.ndarray: - """Return a per-selected-pixel mask of columns valid for one motion bin. - - The valid region is the PSF-dilated reference support minus its - horizontally-eroded border (erosion width ``2 * psf_w - 1``), evaluated at - this bin's motion shift. - """ - plane_size = dmd_pixels_per_column * dmd_pixels_per_row - valid_pix_mask = np.zeros( - (num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row), dtype=bool - ) - valid_pix_mask[ - ref_d, - ref_r + int(unique_motion_to_keep_yx[motion_idx, 0]), - ref_c + int(unique_motion_to_keep_yx[motion_idx, 1]), - ] = True - valid_pix_mask = ndimage.binary_dilation( - valid_pix_mask, - structure=np.ones((1, psf2d.shape[0], psf2d.shape[1]), dtype=bool), - ) - valid_pix_mask = ndimage.binary_erosion( - valid_pix_mask, - # TODO: check if dimensions here should change - structure=np.ones( - (1, psf2d.shape[0], psf2d.shape[1] * 2 - 1), dtype=bool - ), - ) - valid_pix_idxs = np.flatnonzero(valid_pix_mask) - valid_lookup = np.zeros(num_fast_zs * plane_size, dtype=bool) - valid_lookup[valid_pix_idxs] = True - return valid_lookup[sel_pix_idxs] - - -def compute_rho( - residual: np.ndarray, - mot_inds_yx: np.ndarray, - unique_motion_to_keep_yx: np.ndarray, - h_mots: List[torch.Tensor], - d_mats: List[torch.Tensor], - d_mats_expanded: List[torch.Tensor], - sel_pix_idxs: np.ndarray, - ref_d: np.ndarray, - ref_r: np.ndarray, - ref_c: np.ndarray, - num_fast_zs: int, - dmd_pixels_per_column: int, - dmd_pixels_per_row: int, - psf2d: np.ndarray, - verbose: bool = False, -) -> np.ndarray: - """Compute the ``rho`` matched-filter response on the selected pixels. - - For each motion bin and z-plane, restricts ``H`` to that plane, forms the - normalized ``HD`` and ``HD_expanded`` projections, and projects the - residual onto ``HD - HD_expanded`` over the plane's valid columns. - - Parameters - ---------- - residual : ndarray of shape (n_superpixels, n_frames) - Noise-normalized residual from :func:`compute_residual`. - mot_inds_yx : ndarray of shape (n_frames,) - Per-frame motion-bin index (``-1`` for dropped frames). - unique_motion_to_keep_yx : ndarray of shape (n_yx_bins, 2) - Kept 2-D motion vectors. - h_mots : list of torch.Tensor - Per-motion sparse ``H`` from :func:`build_motion_h_matrices`. - d_mats, d_mats_expanded : list of torch.Tensor - Per-plane base/expanded convolution matrices from - :func:`build_convolution_matrix`. - sel_pix_idxs : ndarray of int - Sorted flat selected-pixel indices. - ref_d, ref_r, ref_c : ndarray - Per-superpixel reference depth/row/column indices. - num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row : int - Grid geometry. - psf2d : ndarray - The (cropped) PSF for this DMD. - verbose : bool - Show a per-motion-bin progress bar when set. - - Returns - ------- - ndarray of shape (n_sel, n_frames), float32 - The ``rho`` response (NaN where not computed). - """ - plane_size = dmd_pixels_per_column * dmd_pixels_per_row - z_masks, z_col_idxs, z_remaps = _plane_column_maps( - sel_pix_idxs, num_fast_zs, plane_size - ) - z_col_idxs_np = [zi.numpy() for zi in z_col_idxs] - - rho = np.full( - (len(sel_pix_idxs), residual.shape[1]), np.nan, dtype=np.float32 - ) - - for i in progress( - range(len(unique_motion_to_keep_yx)), - desc="Computing rho", - verbose=verbose, - ): - motion_frames = np.flatnonzero(mot_inds_yx == i) - if motion_frames.size == 0: - continue - - h_mot = h_mots[i].coalesce() - h_idxs = h_mot.indices() - h_vals = h_mot.values() - nrows, _ = h_mot.size() - residual_motion = residual[:, motion_frames].T - - valid_sel_cols = _valid_sel_cols_for_motion( - i, - unique_motion_to_keep_yx, - ref_d, - ref_r, - ref_c, - sel_pix_idxs, - num_fast_zs, - dmd_pixels_per_column, - dmd_pixels_per_row, - psf2d, - ) - - for z in range(num_fast_zs): - new_ncols = int(z_col_idxs[z].numel()) - if new_ncols == 0: - continue - - z_mask = z_masks[z] - keep_mask = z_mask[h_idxs[1]] - if keep_mask.sum().item() == 0: - # Unreachable defensive guard (mirrors the reference): a plane - # with selected pixels (new_ncols > 0) always has H columns in - # it, since both derive from the same in-plane PSF footprint. - continue # pragma: no cover - - remap = z_remaps[z] - new_rows = h_idxs[0, keep_mask] - new_cols = remap[h_idxs[1, keep_mask]] - new_idxs = torch.stack([new_rows, new_cols], dim=0) - new_vals = h_vals[keep_mask] - h_sub = torch.sparse_coo_tensor( - new_idxs, new_vals, (nrows, new_ncols), dtype=torch.float32 - ).coalesce() - - hd = torch.sparse.mm(h_sub, d_mats[z]) - hd = hd / torch.sum(hd, dim=0, keepdim=True) - # Center-surround matched filter. Both projections are - # column-normalized before subtracting, so ``hd_diff`` has - # zero-sum columns and is blind to spatially flat residual. - hd_expanded = torch.sparse.mm(h_sub, d_mats_expanded[z]) - hd_expanded = hd_expanded / torch.sum( - hd_expanded, dim=0, keepdim=True - ) - hd_diff = hd - hd_expanded - # TEMPORARY: whiten each filter column to unit L2 norm so every - # selected pixel's rho has the same noise variance. The L1 - # normalization above pins each column's *sum*, not its norm, so - # noise gain varies with where a column sits relative to the - # sparse superpixel reference-pixel lattice. Since the activity - # image accumulates rho^2 only at local maxima, that bias is - # amplified into a visible grid of bright reference pixels. - # Whitening also makes ``peak_th`` a true sigma threshold. - # To revert, delete the two lines below. - hd_norm = torch.linalg.norm(hd_diff, dim=0, keepdim=True) - hd_diff = hd_diff / hd_norm.clamp_min(1e-12) - - z_cols = z_col_idxs_np[z] - z_valid_mask = valid_sel_cols[z_cols] - if not np.any(z_valid_mask): - continue - z_valid_cols = z_cols[z_valid_mask] - local_valid_cols = remap[torch.from_numpy(z_valid_cols)].long() - - rho[np.ix_(z_valid_cols, motion_frames)] = ( - residual_motion @ hd_diff[:, local_valid_cols].numpy() - ).T - - return rho - - -def mask_high_nan_rho(rho: np.ndarray, thresh: float = 0.5) -> np.ndarray: - """NaN-out rows of ``rho`` that are mostly NaN and return the NaN fraction. - - ``rho`` is modified in place. Rows whose NaN fraction exceeds ``thresh`` - are set entirely to NaN. - - Parameters - ---------- - rho : ndarray of shape (n_sel, n_frames) - The rho response. - thresh : float - NaN-fraction threshold above which a row is dropped. - - Returns - ------- - ndarray of shape (n_sel,) - Per-row NaN fraction (before dropping). - """ - nan_ct = np.mean(np.isnan(rho), axis=1) - rho[nan_ct > thresh] = np.nan - return nan_ct - - -def decay_kernel_1d(decay_tau_s: float, align_hz: float) -> np.ndarray: - """Build the normalized causal exponential-decay smoothing kernel. - - Parameters - ---------- - decay_tau_s : float - Calcium-decay time constant, in seconds. - align_hz : float - Alignment/analysis rate, in Hz. - - Returns - ------- - ndarray - A 1-D kernel spanning ``3 * tau`` frames, summing to 1. - """ - decay_tau_frames = decay_tau_s * align_hz - k1d = np.exp( - np.linspace( - -np.ceil(decay_tau_frames * 3), - 0, - int(np.ceil(decay_tau_frames * 3) + 1), - ) - / decay_tau_frames - ) - return k1d / np.sum(k1d) - - -def smooth_rho( - rho: np.ndarray, k1d: np.ndarray, verbose: bool = False -) -> np.ndarray: - """NaN-aware temporal smoothing of ``rho`` with a 1-D decay kernel. - - Convolves along time only, dividing the smoothed values by the smoothed - valid-sample count and NaN-ing samples whose effective count drops below - ``0.75``. Rows are processed in chunks to bound temporary memory. - ``rho`` is modified in place and also returned. - - Parameters - ---------- - rho : ndarray of shape (n_sel, n_frames) - The rho response (NaN allowed). - k1d : ndarray - 1-D smoothing kernel from :func:`decay_kernel_1d`. - verbose : bool - Show a per-row-chunk progress bar when set. - - Returns - ------- - ndarray - The smoothed ``rho``. - """ - k2d = np.expand_dims(k1d, 0) - n_time = rho.shape[1] - bytes_per_row = max(1, n_time) * np.dtype(np.float32).itemsize * 3 - row_chunk = max(64, min(4096, int(200_000_000 // bytes_per_row))) - - for r0 in progress( - range(0, rho.shape[0], row_chunk), - desc="Smoothing rho", - verbose=verbose, - ): - r1 = min(r0 + row_chunk, rho.shape[0]) - rc = rho[r0:r1].copy() - row_has_data = np.any(np.isfinite(rc), axis=1) - if not np.any(row_has_data): - rho[r0:r1] = rc - continue - - rc_valid = rc[row_has_data] - rho_num = np.nan_to_num(rc_valid, nan=0.0) - rho_den = signal.convolve( - np.isfinite(rc_valid).astype(np.float32), k2d, mode="same" - ) - rho_num = signal.convolve(rho_num, k2d, mode="same") - valid_den = rho_den > 0.75 - np.divide(rho_num, rho_den, out=rho_num, where=valid_den) - rho_num[~valid_den] = np.nan - rc[row_has_data] = rho_num - rho[r0:r1] = rc - - return rho diff --git a/src/giant_python/bandsilo/geometry.py b/src/giant_python/bandsilo/geometry.py deleted file mode 100644 index 92e04e5..0000000 --- a/src/giant_python/bandsilo/geometry.py +++ /dev/null @@ -1,364 +0,0 @@ -"""DMD geometry, reference-stack/PSF loading, and sparse projection matrices. - -Ported from the setup blocks of ``extractSLAP2IntegrationSources.py``: -``ref_pixs_to_drc`` (flat pixel -> depth/column/row), the band-registration -lookup-table reader, ``subsampleMatrixInds`` construction, reference-stack -loading, PSF loading/cropping (from bundled ``assets/psfs/dil-NN.tif``), and -the sparse ``H`` PSF-convolution matrix used to project image space into -superpixel space. -""" - -from __future__ import annotations - -from importlib import resources -from pathlib import Path -from typing import Optional, Tuple, Union - -import numpy as np -import tifffile - -from .hdf5 import load_struct_from_h5 - - -def ref_pixs_to_drc( - ref_pixs: np.ndarray, - dmd_pixels_per_column: int, - dmd_pixels_per_row: int, -) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: - """Map flat reference-pixel indices to DMD (depth, column, row) indices. - - Parameters - ---------- - ref_pixs : ndarray of int - Flat reference-pixel indices. - dmd_pixels_per_column, dmd_pixels_per_row : int - DMD geometry. - - Returns - ------- - ref_d, ref_c, ref_r : ndarray of int32 - Depth, column, and row indices. - """ - ref_pixs = np.asarray(ref_pixs, dtype=np.int64) - plane = int(dmd_pixels_per_column) * int(dmd_pixels_per_row) - npc = int(dmd_pixels_per_column) - ref_d = np.floor_divide(ref_pixs, plane).astype(np.int32) - ref_c = np.floor_divide( - ref_pixs - ref_d.astype(np.int64) * plane, npc - ).astype(np.int32) - ref_r = np.mod(ref_pixs, npc).astype(np.int32) - return ref_d, ref_c, ref_r - - -def load_lookup_table(path: Union[str, Path], n_dmds: int) -> dict: - """Load the band-registration lookup table (``bandRegLookupTable.h5``). - - Parameters - ---------- - path : str or Path - Path to the lookup-table HDF5 file. - n_dmds : int - Number of DMD paths. - - Returns - ------- - dict - ``{"allSuperPixelIDs": {...}, "sparseMaskInds": {...}, - "fastZ2RefZ": {...}}``, each an inner dict keyed ``DMD{N}``. - """ - lt = load_struct_from_h5(path) - all_super_pixel_ids = {} - sparse_mask_inds = {} - fastz_to_refz = {} - for d in range(n_dmds): - p = lt.get(f"Path{d + 1}", lt.get(f"DMD{d + 1}")) - key = f"DMD{d + 1}" - all_super_pixel_ids[key] = ( - np.asarray(p["allSuperPixelIDs"]).reshape(-1, 1).astype(np.int32) - ) - sparse_mask_inds[key] = np.asarray(p["sparseMaskInds"]).astype( - np.int32 - ) - fastz_to_refz[key] = ( - np.asarray(p["fastZ2RefZ"]).reshape(-1, 1).astype(np.int32) - ) - return { - "allSuperPixelIDs": all_super_pixel_ids, - "sparseMaskInds": sparse_mask_inds, - "fastZ2RefZ": fastz_to_refz, - } - - -def build_subsample_matrix_inds( - all_super_pixel_ids: np.ndarray, - sparse_mask_inds: np.ndarray, -) -> np.ndarray: - """Build the superpixel -> reference-pixel index map for one DMD. - - For each superpixel, picks a reference open pixel: if the number of open - pixels is odd, the median pixel value is used; if even, the middle element - (by sparse-mask order) is used. - - Parameters - ---------- - all_super_pixel_ids : ndarray of shape (n_superpixels, 1) - Superpixel ids for this DMD. - sparse_mask_inds : ndarray of shape (N, 2) - ``[open_pixel (1-based), superpixel_id (1-based)]`` rows. - - Returns - ------- - ndarray of shape (n_superpixels, 2), int32 - ``[ref_pixel (0-based), superpixel_id (1-based)]`` per superpixel. - """ - num_super_pixels = all_super_pixel_ids.shape[0] - out = np.zeros((num_super_pixels, 2), dtype=np.int32) - for sp in range(num_super_pixels): - inds = np.where(sparse_mask_inds[:, 1] == sp + 1)[0] - open_pixs = sparse_mask_inds[inds, 0] - 1 - ref_pix = open_pixs[int(np.floor(len(open_pixs) / 2))] - out[sp, 0] = ref_pix - out[sp, 1] = sp + 1 - return out - - -def _ref_stack_group(ref_stack: dict, dmd_ix: int): - """Return the ref_stack subgroup for a DMD (``Path{N}`` or ``DMD{N}``).""" - return ref_stack.get( - f"Path{dmd_ix + 1}", ref_stack.get(f"DMD{dmd_ix + 1}") - ) - - -def find_reference_file( - datadr: Union[str, Path], dmd_ix: int -) -> Optional[str]: - """Find the REFERENCE tif for a DMD under ``datadr`` (recursive glob). - - Parameters - ---------- - datadr : str or Path - Raw-data directory. - dmd_ix : int - 0-based DMD index. - - Returns - ------- - str or None - Path to the first matching REFERENCE tif, or ``None`` if none found. - """ - patterns = ( - f"**/*DMD{dmd_ix + 1}_CONFIG2-REFERENCE*", - f"**/*DMD{dmd_ix + 1}-REFERENCE*", - ) - for pattern in patterns: - matches = list(Path(datadr).glob(pattern)) - if matches: - return str(matches[0]) - return None - - -def load_reference_stack( - datadr: Union[str, Path], - ref_stack_meta: dict, - dmd_ix: int, -) -> Tuple[Optional[np.ndarray], np.ndarray, Optional[str]]: - """Load the band reference stack for one DMD. - - Reads the REFERENCE tif from ``datadr`` but takes the channel list from the - trial table's embedded ref_stack metadata. The raw counts are scaled by - ``1/100`` and reshaped to ``[channels, z, y, x]``. - - Parameters - ---------- - datadr : str or Path - Raw-data directory. - ref_stack_meta : dict - The trial table's ``ref_stack`` sub-struct. - dmd_ix : int - 0-based DMD index. - - Returns - ------- - ref_stack : ndarray or None - ``[channels, z, y, x]`` stack, or ``None`` if no REFERENCE file found. - channels : ndarray - Channel indices for this DMD. - ref_file : str or None - The REFERENCE file used, or ``None``. - """ - grp = _ref_stack_group(ref_stack_meta, dmd_ix) - channels = np.asarray(grp["channels"]).reshape(-1) - num_channels = len(channels) - - ref_file = find_reference_file(datadr, dmd_ix) - if ref_file is None: - return None, channels, None - - ref = tifffile.imread(ref_file) / 100 - ref = ref.reshape(-1, num_channels, ref.shape[1], ref.shape[2]) - ref = ref.transpose(1, 0, 2, 3) - return ref, channels, ref_file - - -def default_psf(dilation: int) -> np.ndarray: - """Load a bundled default PSF template (``assets/psfs/dil-NN.tif``). - - Parameters - ---------- - dilation : int - Dilation size selecting ``dil-{dilation:02d}.tif`` (e.g. ``17``). - - Returns - ------- - ndarray of float32 - The PSF template image. - """ - name = f"dil-{int(dilation):02d}.tif" - resource = ( - resources.files("giant_python") - .joinpath("assets") - .joinpath("psfs") - .joinpath(name) - ) - with resources.as_file(resource) as p: - return tifffile.imread(str(p)).astype(np.float32) - - -def _pad_to(psf2d: np.ndarray, height: int, width: int) -> np.ndarray: - """Center-pad a 2-D PSF to ``(height, width)`` with its minimum value.""" - ph, pw = psf2d.shape - pad_r = (height - ph) // 2 - pad_c = (width - pw) // 2 - return np.pad( - psf2d, - ((pad_r, pad_r), (pad_c, pad_c)), - constant_values=np.min(psf2d), - ) - - -def build_combined_psf(psfs: list) -> np.ndarray: - """Stack per-DMD PSFs into a ``(n_dmds, H, W)`` array, center-padded.""" - height = max(p.shape[0] for p in psfs) - width = max(p.shape[1] for p in psfs) - combined = np.zeros((len(psfs), height, width), dtype=np.float32) - for i, p in enumerate(psfs): - combined[i] = _pad_to(p, height, width) - return combined - - -def threshold_and_crop_psf(psf2d: np.ndarray) -> np.ndarray: - """Zero values below ``max * exp(-3)`` and crop boundary zeros. - - Parameters - ---------- - psf2d : ndarray - A single-DMD PSF image. - - Returns - ------- - ndarray of float32 - The thresholded, tightly-cropped PSF. - """ - psf2d = psf2d.astype(np.float32, copy=True) - psf2d[psf2d < np.max(psf2d) * np.exp(-3)] = 0 - non_zero_rows = np.any(psf2d != 0, axis=1) - non_zero_cols = np.any(psf2d != 0, axis=0) - row_start, row_end = np.where(non_zero_rows)[0][[0, -1]] - col_start, col_end = np.where(non_zero_cols)[0][[0, -1]] - row_stop = row_end + 1 - col_stop = col_end + 1 - return psf2d[row_start:row_stop, col_start:col_stop] - - -def load_psf(dilation: int, n_dmds: int) -> dict: - """Load per-DMD PSFs from bundled assets, thresholded and cropped. - - Each DMD uses the same ``dil-NN.tif`` template from - ``assets/psfs/``, center-padded to a common size when shapes differ. - - Parameters - ---------- - dilation : int - Dilation size selecting the bundled PSF template. - n_dmds : int - Number of DMD paths. - - Returns - ------- - dict - ``{"DMD{N}": psf2d}`` thresholded/cropped PSFs. - """ - psfs = [default_psf(dilation) for _ in range(n_dmds)] - psf_combined = build_combined_psf(psfs) - return { - f"DMD{i + 1}": threshold_and_crop_psf(psf_combined[i]) - for i in range(n_dmds) - } - - -def build_sparse_h( - subsample_matrix_inds: np.ndarray, - psf2d: np.ndarray, - dmd_pixels_per_column: int, - dmd_pixels_per_row: int, -) -> Tuple[np.ndarray, np.ndarray]: - """Build the sparse ``H`` PSF-convolution matrix (COO index/value arrays). - - ``H`` projects image space (flattened ``z*(rows*cols) + row*cols + col``) - into superpixel space by convolving each superpixel's reference pixel with - the PSF. - - Parameters - ---------- - subsample_matrix_inds : ndarray of shape (n_superpixels, 2) - ``[ref_pixel (0-based), superpixel_id (1-based)]`` per superpixel. - psf2d : ndarray - The (thresholded/cropped) PSF for this DMD. - dmd_pixels_per_column, dmd_pixels_per_row : int - DMD plane geometry. - - Returns - ------- - sparse_h_inds : ndarray of shape (2, nnz), int32 - Row (superpixel) / column (image-pixel) indices of non-zero entries. - sparse_h_vals : ndarray of shape (nnz,), float32 - The corresponding PSF weights. - """ - ref_d, ref_c, ref_r = ref_pixs_to_drc( - subsample_matrix_inds[:, 0], dmd_pixels_per_column, dmd_pixels_per_row - ) - psf2d = np.asarray(psf2d, dtype=np.float32) - psf_h, psf_w = psf2d.shape - filter_size = psf_h * psf_w - plane_size = dmd_pixels_per_column * dmd_pixels_per_row - num_super_pixels = subsample_matrix_inds.shape[0] - - row_offsets = np.arange(psf_h, dtype=np.int32) - (psf_h // 2) - col_offsets = np.arange(psf_w, dtype=np.int32) - (psf_w // 2) - row_offsets_flat = np.broadcast_to( - row_offsets.reshape(-1, 1), (psf_h, psf_w) - ).ravel() - col_offsets_flat = np.broadcast_to( - col_offsets.reshape(1, -1), (psf_h, psf_w) - ).ravel() - psf_vals_flat = psf2d.ravel() - - sparse_h_inds = np.zeros( - (2, num_super_pixels * filter_size), dtype=np.int32 - ) - sparse_h_vals = np.zeros( - (num_super_pixels * filter_size,), dtype=np.float32 - ) - sparse_h_inds[0] = np.repeat(subsample_matrix_inds[:, 1] - 1, filter_size) - for sp in range(num_super_pixels): - start = sp * filter_size - end = (sp + 1) * filter_size - rows = ref_r[sp] + row_offsets_flat - cols = ref_c[sp] + col_offsets_flat - sparse_h_inds[1, start:end] = ( - ref_d[sp] * plane_size + rows * dmd_pixels_per_row + cols - ).astype(np.int32, copy=False) - sparse_h_vals[start:end] = psf_vals_flat - - non_zero_mask = sparse_h_vals != 0 - return sparse_h_inds[:, non_zero_mask], sparse_h_vals[non_zero_mask] diff --git a/src/giant_python/bandsilo/gui.py b/src/giant_python/bandsilo/gui.py deleted file mode 100644 index a123cdb..0000000 --- a/src/giant_python/bandsilo/gui.py +++ /dev/null @@ -1,694 +0,0 @@ -"""Parameter GUI, user-ROI annotation GUI, and annotation HDF5 IO. - -Ported from ``extractSLAP2IntegrationSources.py`` (parameter dialog ref -1013-1152, annotation helpers/IO ref 1155-1357, user-ROI selection GUI ref -1567-1741). The interactive dialogs (``tkinter`` parameter form, ``tkinter`` -label prompt, ``cv2`` ROI selector) import their GUI toolkits lazily so that -importing this module never requires a display; they are excluded from coverage -because they cannot run headless. The pure geometry/superpixel helpers and the -annotation reader/writer are fully testable and carry the numerical contract. -""" - -from __future__ import annotations - -import os -from dataclasses import replace -from pathlib import Path -from typing import Optional - -import h5py -import numpy as np - -from ..models import SiloParams - -# Mapping of parameter-GUI fields to SiloParams attributes. ``sparse_fac`` is -# handled separately (the GUI exposes its natural log). -_PARAM_GUI_FIELDS = ( - ("analyze_hz", "Analyze Hz:"), - ("decay_tau_s", "Decay Tau (s):"), - ("baseline_window_s", "Baseline Window (s):"), - ("denoise_window_s", "Denoise Window (s):"), - ("vif", "Variance Inflation Factor (VIF):"), - ("d_xy", "dXY:"), - ("peakth", "Peak Threshold:"), - ("peak_buffer", "Peak Buffer:"), - ("max_workers", "Max Workers:"), - ("operator", "Operator:"), -) - - -def _h5_scalar_str(ds: h5py.Dataset) -> str: - """Read an h5 scalar string dataset as a python ``str``.""" - v = ds[()] - if isinstance(v, np.ndarray): - v = v.reshape(-1)[0] - if isinstance(v, bytes): - return v.decode("utf-8", "replace") - return str(v) - - -def user_roi_superpixel_lists_from_masks( - roi_masks: list, - sp_fastz: np.ndarray, - sp_rows: np.ndarray, - sp_cols: np.ndarray, -) -> list: - """Return one superpixel-index list per user ROI, from boolean masks. - - ROIs may overlap: a superpixel inside several ROIs appears in each list. - - Parameters - ---------- - roi_masks : list of ndarray - Per-ROI boolean masks of shape ``(num_fast_z, Y, X)``. - sp_fastz, sp_rows, sp_cols : ndarray - Per-superpixel fast-Z / row / column indices into the mask. - - Returns - ------- - list of ndarray - For each ROI, the flat indices of superpixels falling inside it. - """ - return [ - np.flatnonzero(np.asarray(m)[sp_fastz, sp_rows, sp_cols]) - for m in roi_masks - ] - - -def compute_user_roi_geometry( - ref: np.ndarray, - fastz_to_refz: np.ndarray, - subsample_matrix_inds: np.ndarray, - motion_median: tuple, -) -> dict: - """Compute reference-image geometry for the user-ROI selection GUI. - - Derives the best display channel, the fast-Z -> reference-Z map (shifted by - the median Z motion), and the per-superpixel (fast-Z, row, column) indices - (shifted by the median row/column motion), plus a boolean superpixel mask. - - Parameters - ---------- - ref : ndarray - Reference stack for one DMD, shape ``[channels, z, Y, X]``. - fastz_to_refz : ndarray - Fast-Z -> reference-Z lookup (1-based), shape ``(num_fast_z, 1)``. - subsample_matrix_inds : ndarray - ``[ref_pixel (0-based), superpixel_id]`` rows for this DMD. - motion_median : tuple of int - Median ``(row, column, z)`` motion for this DMD. - - Returns - ------- - dict - Geometry fields consumed by the user-ROI selection GUI and the - annotation reader: ``ref``, ``num_ref_z``, ``yx_shape``, ``best_ch``, - ``z_map``, ``num_fast_z``, ``sp_fastz``, ``sp_rows``, ``sp_cols``, - ``sp_mask``. - """ - avg_motion_r, avg_motion_c, avg_motion_z = motion_median - - num_ref_z = ref.shape[1] - yx_shape = (ref.shape[2], ref.shape[3]) - plane = yx_shape[0] * yx_shape[1] - ch_means = [np.nanmean(ref[c]) for c in range(ref.shape[0])] - best_ch = int(np.argmax(ch_means)) if len(ch_means) > 0 else 0 - - z_map = np.array(fastz_to_refz + avg_motion_z).reshape(-1) - 1 - num_fast_z = z_map.shape[0] - - ref_pix = subsample_matrix_inds[:, 0] - sp_fastz = ref_pix // plane - sp_cols = avg_motion_c + (ref_pix - sp_fastz * plane) // yx_shape[0] - sp_rows = avg_motion_r + ref_pix % yx_shape[0] - - sp_mask = np.zeros((num_fast_z, *yx_shape), dtype=bool) - valid = ( - (sp_rows >= 0) - & (sp_rows < yx_shape[0]) - & (sp_cols >= 0) - & (sp_cols < yx_shape[1]) - & (sp_fastz >= 0) - & (sp_fastz < num_fast_z) - ) - if np.any(valid): - sp_mask[sp_fastz[valid], sp_rows[valid], sp_cols[valid]] = True - - return { - "ref": ref, - "num_ref_z": num_ref_z, - "yx_shape": yx_shape, - "best_ch": best_ch, - "z_map": z_map, - "num_fast_z": num_fast_z, - "sp_fastz": sp_fastz, - "sp_rows": sp_rows, - "sp_cols": sp_cols, - "sp_mask": sp_mask, - } - - -def save_annotations_h5( - dr: str, - roi_records_by_dmd: dict, - user_roi_masks: dict, - n_dmds: int, - ref_files: Optional[dict] = None, -) -> str: - """Write user ROI annotations to ``dr/annotations.h5``. - - Rectangles drawn in the OpenCV selector are stored as ``type='polygon'`` - with a 4-vertex ``position`` (``[y, x]``, 0-indexed) plus a per-ROI binary - ``mask`` (fast-Z, Y, X). ``roi_records_by_dmd[key]`` is a list (in ROI - order) of dicts with keys ``type``, ``label``, ``position``; the per-ROI - mask is taken from ``user_roi_masks[key][i]`` (ROIs may overlap). - - Parameters - ---------- - dr : str - Destination directory (``annotations.h5`` is written inside it). - roi_records_by_dmd : dict - ``{"DMD{N}": [record, ...]}`` ROI metadata. - user_roi_masks : dict - ``{"DMD{N}": [mask, ...]}`` per-ROI boolean masks. - n_dmds : int - Number of DMD paths. - ref_files : dict, optional - ``{"DMD{N}": reference_file_path}`` for provenance. - - Returns - ------- - str - The path to the written ``annotations.h5``. - """ - path = os.path.join(dr, "annotations.h5") - str_dt = h5py.string_dtype(encoding="utf-8") - ref_files = ref_files or {} - with h5py.File(path, "w") as f: - f["row_major"] = 1 - f["coords_zero_indexed"] = 1 - for dmd_ix in range(n_dmds): - dmd_key = f"DMD{dmd_ix + 1}" - grp = f.create_group(f"Path{dmd_ix + 1}") - - ref_file = ref_files.get(dmd_key) - grp.create_dataset( - "dr", - data=(str(Path(ref_file).parent) if ref_file else ""), - dtype=str_dt, - ) - grp.create_dataset( - "fn", - data=(Path(ref_file).name if ref_file else ""), - dtype=str_dt, - ) - - records = roi_records_by_dmd.get(dmd_key, []) - grp.create_dataset("n_rois", data=len(records)) - - mask_list = user_roi_masks.get(dmd_key, []) - for i, rec in enumerate(records): - _write_roi_group(grp, i, rec, mask_list, str_dt) - return path - - -def _write_roi_group( - grp: h5py.Group, - i: int, - rec: dict, - mask_list: list, - str_dt, -) -> None: - """Write a single ``roi_###`` subgroup (type/label/mask/position).""" - rgrp = grp.create_group(f"roi_{i:03d}") - rtype = rec.get("type", "polygon") - rgrp.create_dataset("type", data=rtype, dtype=str_dt) - rgrp.create_dataset( - "label", data=rec.get("label", f"ROI{i + 1}"), dtype=str_dt - ) - if i < len(mask_list): - roi_mask = np.asarray(mask_list[i]).astype(np.uint8) - rgrp.create_dataset( - "mask", data=roi_mask, compression="gzip", shuffle=True - ) - if rtype == "polygon" and "position" in rec: - rgrp.create_dataset( - "position", data=np.asarray(rec["position"], dtype=np.float64) - ) - - -def _empty_path_selection(dmd_key: str, out: dict) -> None: - """Record an empty (no-ROI) selection for one DMD across all outputs.""" - out["user_roi_masks"][dmd_key] = [] - out["user_roi_superpixels"][dmd_key] = [] - out["user_roi_labels"][dmd_key] = [] - out["roi_records"][dmd_key] = [] - - -def _read_roi_records(grp: h5py.Group, expected: tuple): - """Read all ``roi_###`` subgroups for one path. - - Parameters - ---------- - grp : h5py.Group - The ``Path{N}`` group. - expected : tuple - Expected mask shape ``(num_fast_z, Y, X)``. - - Returns - ------- - tuple - ``(roi_masks, labels, recs, bad_shape)`` where ``bad_shape`` is True if - any ROI mask shape did not match ``expected``. - """ - n_rois = 0 - if "n_rois" in grp: - n_rois = int(np.asarray(grp["n_rois"][()]).reshape(-1)[0]) - roi_masks, labels, recs = [], [], [] - for i in range(n_rois): - rgrp = grp.get(f"roi_{i:03d}") - if rgrp is None: - continue - lbl = _h5_scalar_str(rgrp["label"]) if "label" in rgrp else f"ROI{i+1}" - typ = _h5_scalar_str(rgrp["type"]) if "type" in rgrp else "polygon" - if "mask" in rgrp: - m = np.asarray(rgrp["mask"][()]) - if m.shape != expected: - return roi_masks, labels, recs, True - roi_masks.append(m > 0) - else: - roi_masks.append(np.zeros(expected, dtype=bool)) - labels.append(lbl) - rec = {"type": typ, "label": lbl} - if "position" in rgrp: - rec["position"] = np.asarray(rgrp["position"][()]) - recs.append(rec) - return roi_masks, labels, recs, False - - -def load_annotations_h5( - annotations_path: str, - n_dmds: int, - user_roi_geo: dict, -) -> tuple: - """Load user ROI annotations from ``annotations.h5`` when the file exists. - - Reconstructs, per path, the per-ROI superpixel lists, text labels, and raw - ROI records. Each ``Path`` group may be absent (treated as no selection). - If a per-ROI mask shape does not match the current geometry, that path is - treated as no selection. - - Parameters - ---------- - annotations_path : str - Path to ``annotations.h5``. - n_dmds : int - Number of DMD paths. - user_roi_geo : dict - Per-DMD geometry from :func:`compute_user_roi_geometry`. - - Returns - ------- - tuple - ``(skip_manual, user_roi_masks, user_roi_superpixels, - user_roi_labels, roi_records)``. ``skip_manual`` is True iff the file - was read and at least one path had >= 1 ROI. On a missing or unreadable - file, returns ``(False, {}, {}, {}, {})`` and the caller runs manual - selection. - """ - empty = (False, {}, {}, {}, {}) - if not os.path.exists(annotations_path): - return empty - - out = { - "user_roi_masks": {}, - "user_roi_superpixels": {}, - "user_roi_labels": {}, - "roi_records": {}, - } - any_valid = False - try: - with h5py.File(annotations_path, "r") as hf: - for dmd_ix in range(n_dmds): - if _load_one_path(hf, dmd_ix, user_roi_geo, out): - any_valid = True - except OSError as e: - print(f"Could not read {annotations_path}: {e}") - return empty - - if not any_valid: - return empty - return ( - True, - out["user_roi_masks"], - out["user_roi_superpixels"], - out["user_roi_labels"], - out["roi_records"], - ) - - -def _load_one_path( - hf: h5py.File, - dmd_ix: int, - user_roi_geo: dict, - out: dict, -) -> bool: - """Load one path's annotations into ``out``; return True if it had ROIs.""" - dmd_key = f"DMD{dmd_ix + 1}" - path_key = f"Path{dmd_ix + 1}" - geo = user_roi_geo[dmd_key] - expected = (geo["num_fast_z"], geo["yx_shape"][0], geo["yx_shape"][1]) - - grp = hf.get(path_key) - if grp is None: - _empty_path_selection(dmd_key, out) - return False - - roi_masks, labels, recs, bad_shape = _read_roi_records(grp, expected) - if bad_shape: - print( - f"annotations.h5: {path_key} ROI mask shape mismatch " - f"(expected {expected}); treating this path as no selection." - ) - _empty_path_selection(dmd_key, out) - return False - - out["user_roi_masks"][dmd_key] = roi_masks - out["user_roi_superpixels"][ - dmd_key - ] = user_roi_superpixel_lists_from_masks( - roi_masks, geo["sp_fastz"], geo["sp_rows"], geo["sp_cols"] - ) - out["user_roi_labels"][dmd_key] = labels - out["roi_records"][dmd_key] = recs - if len(recs) > 0: - print(f"Loaded {len(recs)} ROI(s) for {path_key} from annotations.h5") - return True - return False - - -def run_parameter_gui( - params: Optional[SiloParams] = None, -) -> Optional[SiloParams]: # pragma: no cover - interactive tkinter dialog - """Open the tkinter parameter form and return an updated ``SiloParams``. - - The form is pre-filled from ``params`` (or :class:`SiloParams` defaults). - ``sparse_fac`` is edited on a natural-log scale. Returns ``None`` if the - user cancels. - - Parameters - ---------- - params : SiloParams, optional - Parameters to pre-fill; defaults to a fresh :class:`SiloParams`. - - Returns - ------- - SiloParams or None - The updated parameters, or ``None`` if cancelled. - """ - import tkinter as tk - from tkinter import messagebox, ttk - - base = params or SiloParams() - root = tk.Tk() - root.title("SLAP2 Analysis Parameters") - frame = ttk.Frame(root, padding="10") - frame.grid(row=0, column=0) - - entries = {} - row = 0 - for attr, label in _PARAM_GUI_FIELDS: - ttk.Label(frame, text=label).grid(row=row, column=0, sticky=tk.W) - var = tk.StringVar(value=str(getattr(base, attr))) - ttk.Entry(frame, textvariable=var, width=15).grid(row=row, column=1) - entries[attr] = var - row += 1 - - ttk.Label(frame, text="Sparse Factor (log):").grid( - row=row, column=0, sticky=tk.W - ) - sparse_var = tk.StringVar(value=str(float(np.log(base.sparse_fac)))) - ttk.Entry(frame, textvariable=sparse_var, width=15).grid(row=row, column=1) - row += 1 - - ttk.Label(frame, text="Draw User ROIs?").grid( - row=row, column=0, sticky=tk.W - ) - draw_rois_var = tk.BooleanVar(value=base.draw_user_rois) - ttk.Checkbutton(frame, variable=draw_rois_var).grid(row=row, column=1) - row += 1 - - result = {"params": None} - - def on_ok(): - """Validate entries and store an updated SiloParams, then close.""" - try: - updates = { - "analyze_hz": float(entries["analyze_hz"].get()), - "decay_tau_s": float(entries["decay_tau_s"].get()), - "baseline_window_s": float(entries["baseline_window_s"].get()), - "denoise_window_s": float(entries["denoise_window_s"].get()), - "vif": float(entries["vif"].get()), - "d_xy": int(entries["d_xy"].get()), - "peakth": float(entries["peakth"].get()), - "peak_buffer": int(entries["peak_buffer"].get()), - "max_workers": int(entries["max_workers"].get()), - "operator": entries["operator"].get(), - "sparse_fac": float(np.exp(float(sparse_var.get()))), - "draw_user_rois": bool(draw_rois_var.get()), - } - result["params"] = replace(base, **updates) - root.destroy() - except ValueError as e: - messagebox.showerror("Error", f"Invalid parameter values: {e}") - - def on_cancel(): - """Discard edits and close the dialog.""" - root.destroy() - - button_frame = ttk.Frame(frame) - button_frame.grid(row=row, column=0, columnspan=2, pady=10) - ttk.Button(button_frame, text="OK", command=on_ok).pack(side=tk.LEFT) - ttk.Button(button_frame, text="Cancel", command=on_cancel).pack( - side=tk.LEFT - ) - - root.grab_set() - root.mainloop() - return result["params"] - - -def _ask_roi_label( - roi_num: int, path_num: int, plane_num: int -) -> str: # pragma: no cover - interactive tkinter dialog - """Modal tkinter prompt for one ROI's label; defaults to ``ROI{n}``.""" - import tkinter as tk - from tkinter import simpledialog - - text = None - try: - root = tk.Tk() - root.withdraw() - root.attributes("-topmost", True) - text = simpledialog.askstring( - "ROI label", - f"Label for ROI {roi_num} (Path {path_num}, " - f"fast-Z plane {plane_num}):", - parent=root, - ) - root.destroy() - except Exception as e: # noqa: BLE001 - GUI best-effort - print(f"Could not prompt for ROI label ({e}); using default.") - if not text or not text.strip(): - return f"ROI{roi_num}" - return text.strip() - - -def run_user_roi_selection( - user_roi_geo: dict, - n_dmds: int, -) -> dict: # pragma: no cover - interactive cv2 GUI - """Interactively select user ROIs on each DMD's reference image. - - Scrolls through fast-Z planes and lets the user draw rectangles (stored as - polygons). Returns per-DMD ``user_roi_masks``, ``user_roi_superpixels``, - ``user_roi_labels``, and ``roi_records`` for writing via - :func:`save_annotations_h5`. - - Parameters - ---------- - user_roi_geo : dict - Per-DMD geometry from :func:`compute_user_roi_geometry`. - n_dmds : int - Number of DMD paths. - - Returns - ------- - dict - ``{"user_roi_masks", "user_roi_superpixels", "user_roi_labels", - "roi_records"}``. - """ - import cv2 - - out = { - "user_roi_masks": {}, - "user_roi_superpixels": {}, - "user_roi_labels": {}, - "roi_records": {}, - } - for dmd_ix in range(n_dmds): - dmd_key = f"DMD{dmd_ix + 1}" - g = user_roi_geo[dmd_key] - roi_masks, roi_records = _select_rois_for_dmd(cv2, dmd_ix, g) - out["user_roi_masks"][dmd_key] = roi_masks - out["user_roi_superpixels"][ - dmd_key - ] = user_roi_superpixel_lists_from_masks( - roi_masks, g["sp_fastz"], g["sp_rows"], g["sp_cols"] - ) - out["user_roi_labels"][dmd_key] = [r["label"] for r in roi_records] - out["roi_records"][dmd_key] = roi_records - return out - - -def _normalize_plane( - plane: np.ndarray, -) -> np.ndarray: # pragma: no cover - display helper - """Percentile-normalize a reference plane to an 8-bit image.""" - im = np.nan_to_num(plane, nan=0.0) - vmin = np.percentile(im, 1) - vmax = np.percentile(im, 99.5) - if not np.isfinite(vmin): - vmin = float(np.nanmin(im)) if np.any(np.isfinite(im)) else 0.0 - if not np.isfinite(vmax): - vmax = float(np.nanmax(im)) if np.any(np.isfinite(im)) else 1.0 - if vmax <= vmin: - vmax = vmin + 1.0 - im8 = np.clip((im - vmin) / (vmax - vmin), 0, 1) - return (im8 * 255).astype(np.uint8) - - -def _select_rois_for_dmd( - cv2, dmd_ix: int, g: dict -): # pragma: no cover - interactive cv2 GUI - """Run the interactive ROI selector for one DMD; return masks + records.""" - ref = g["ref"] - num_ref_z = g["num_ref_z"] - yx_shape = g["yx_shape"] - best_ch = g["best_ch"] - z_map = g["z_map"] - num_fast_z = g["num_fast_z"] - sp_mask = g["sp_mask"] - - roi_masks = [] - roi_records = [] - roi_union = np.zeros((num_fast_z, *yx_shape), dtype=bool) - - window_name = f"Select user ROI(s) DMD{dmd_ix + 1}" - cv2.namedWindow(window_name, cv2.WINDOW_NORMAL) - cv2.resizeWindow(window_name, 800, 500) - has_z_trackbar = num_fast_z > 1 - if has_z_trackbar: - cv2.createTrackbar("z", window_name, 0, num_fast_z - 1, lambda v: None) - - curr_fz = 0 - while True: - if has_z_trackbar: - curr_fz = int( - np.clip( - cv2.getTrackbarPos("z", window_name), 0, num_fast_z - 1 - ) - ) - refz = int(np.clip(z_map[curr_fz], 0, max(0, num_ref_z - 1))) - im8 = _normalize_plane(ref[best_ch, refz]) - disp = _compose_display(cv2, im8, sp_mask[curr_fz], roi_union[curr_fz]) - cv2.imshow(window_name, disp) - - keycode = cv2.waitKey(50) & 0xFF - if keycode == ord("e"): - _edit_rois( - cv2, - im8, - sp_mask[curr_fz], - curr_fz, - dmd_ix, - num_fast_z, - yx_shape, - roi_masks, - roi_union, - roi_records, - ) - elif keycode == ord("n"): - curr_fz = min(curr_fz + 1, num_fast_z - 1) - if has_z_trackbar: - cv2.setTrackbarPos("z", window_name, curr_fz) - elif keycode == ord("p"): - curr_fz = max(curr_fz - 1, 0) - if has_z_trackbar: - cv2.setTrackbarPos("z", window_name, curr_fz) - elif keycode in (27, ord("q")): - break - - cv2.destroyWindow(window_name) - return roi_masks, roi_records - - -def _compose_display( - cv2, im8: np.ndarray, sp_plane: np.ndarray, roi_plane: np.ndarray -): # pragma: no cover - display helper - """Overlay superpixel (green) and ROI-union (red) masks on the plane.""" - disp = cv2.cvtColor(im8, cv2.COLOR_GRAY2BGR) - if np.any(sp_plane): - sp_color = np.zeros_like(disp) - sp_color[sp_plane] = (0, 255, 0) - disp = cv2.addWeighted(disp, 0.75, sp_color, 0.25, 0) - if roi_plane.any(): - roi_color = np.zeros_like(disp) - roi_color[roi_plane] = (0, 0, 255) - disp = cv2.addWeighted(disp, 0.7, roi_color, 0.3, 0) - return disp - - -def _edit_rois( - cv2, - im8, - sp_plane, - curr_fz, - dmd_ix, - num_fast_z, - yx_shape, - roi_masks, - roi_union, - roi_records, -): # pragma: no cover - interactive cv2 GUI - """Prompt the user to draw ROI rectangles and append them in place.""" - edit_name = f"Edit ROIs z={curr_fz + 1}" - edit_disp = _compose_display( - cv2, im8, sp_plane, np.zeros(yx_shape, dtype=bool) - ) - rois = cv2.selectROIs( - edit_name, edit_disp, showCrosshair=True, fromCenter=False - ) - cv2.resizeWindow(edit_name, 800, 500) - if rois is not None and len(rois) > 0: - for x, y, w, h in rois: - roi_num = len(roi_masks) + 1 - y_stop, x_stop = y + h, x + w - m = np.zeros((num_fast_z, *yx_shape), dtype=bool) - m[curr_fz, y:y_stop, x:x_stop] = True - roi_masks.append(m) - roi_union[curr_fz, y:y_stop, x:x_stop] = True - label_text = _ask_roi_label(roi_num, dmd_ix + 1, curr_fz + 1) - position = np.array( - [[y, x], [y, x + w], [y + h, x + w], [y + h, x]], - dtype=np.float64, - ) - roi_records.append( - { - "type": "polygon", - "label": label_text, - "position": position, - } - ) - print(f"Added {len(rois)} ROI(s) at fast-Z {curr_fz + 1}") - cv2.destroyWindow(edit_name) diff --git a/src/giant_python/bandsilo/hdf5.py b/src/giant_python/bandsilo/hdf5.py deleted file mode 100644 index 41985a3..0000000 --- a/src/giant_python/bandsilo/hdf5.py +++ /dev/null @@ -1,450 +0,0 @@ -"""HDF5 struct (de)serialization for the BandSILo backend. - -BandRegistration.m (GIAnT-MATLAB) writes its outputs as HDF5 via -``saveStructToH5.m`` rather than the old ``.mat`` files. The readers below -mirror ``loadStructFromH5.m``: groups become nested dicts, string datasets -become ``str`` (or object arrays), and numeric datasets are returned in MATLAB -``size()`` orientation. MATLAB writes column-major (the ``/row_major`` flag is -``0`` or absent), so h5py reads datasets with axes reversed and we transpose to -recover the original layout. - -Ported from ``extractSLAP2IntegrationSources.py`` (``load_struct_from_h5``, -``load_alignment_data_h5``, ``_write_dict_to_h5group``, ``to_serializable``). -This module is intentionally private to :mod:`giant_python.bandsilo`; the -generic ``giant_python.io.hdf5`` helpers remain a separate concern. -""" - -from __future__ import annotations - -import os -from pathlib import Path -from typing import Union - -import h5py -import numpy as np - -from ..models import TrialTable - - -def to_serializable(val): - """Recursively convert numpy types/arrays to JSON-friendly Python values. - - Parameters - ---------- - val : object - A scalar, numpy value/array, or (possibly nested) dict/list/tuple. - - Returns - ------- - object - The same structure with numpy scalars converted to Python scalars and - numpy arrays converted to (nested) lists. - """ - if isinstance(val, dict): - return {k: to_serializable(v) for k, v in val.items()} - if isinstance(val, (list, tuple)): - return type(val)(to_serializable(v) for v in val) - if isinstance(val, np.ndarray): - return val.tolist() - if isinstance(val, np.generic): - return val.item() - return val - - -def _h5_is_string_dataset(ds: h5py.Dataset) -> bool: - """Return whether an h5py dataset holds (fixed/variable-length) strings.""" - if h5py.check_string_dtype(ds.dtype) is not None: - return True - return ds.dtype.kind in ("S", "U", "O") - - -def _decode_one(x) -> str: - """Decode a single bytes/str scalar to ``str``.""" - if isinstance(x, (bytes, np.bytes_)): - return x.decode("utf-8", "replace") - return str(x) - - -def _decode_h5_strings(val): - """Decode a scalar or ndarray of (variable-length) strings to ``str``. - - Parameters - ---------- - val : bytes, str, or ndarray - Raw value read from a string dataset. - - Returns - ------- - str or ndarray of object - Decoded scalar string, or an object array of decoded strings. - """ - if isinstance(val, (bytes, str, np.bytes_, np.str_)): - return _decode_one(val) - arr = np.asarray(val, dtype=object) - if arr.size == 0: - return arr - return np.vectorize(_decode_one, otypes=[object])(arr) - - -def _read_h5_dataset(ds: h5py.Dataset, row_major: bool): - """Read one dataset in MATLAB ``size()`` orientation. - - Scalar strings/paths (e.g. ``datadr``) are returned as plain ``str``; - string grids (e.g. ``filename``, ``fn_adata``) stay as object arrays with - MATLAB shape. - - Parameters - ---------- - ds : h5py.Dataset - The dataset to read. - row_major : bool - Whether the file was written row-major (``True``) or column-major. - - Returns - ------- - object - The decoded value (``str``, scalar, or ndarray). - """ - val = ds[()] - if _h5_is_string_dataset(ds): - val = _decode_h5_strings(val) - if isinstance(val, np.ndarray): - if (not row_major) and val.ndim >= 2: - val = val.T - if val.size == 1: - return val.reshape(-1)[0] - return val - val = np.asarray(val) - if (not row_major) and val.ndim >= 2: - val = val.T - return val - - -def _read_h5_group(grp: h5py.Group, row_major: bool) -> dict: - """Recursively read an h5py group into a nested dict.""" - out = {} - for key, item in grp.items(): - if key == "row_major": - continue - if isinstance(item, h5py.Group): - out[key] = _read_h5_group(item, row_major) - else: - out[key] = _read_h5_dataset(item, row_major) - return out - - -def load_struct_from_h5(path: Union[str, Path]) -> dict: - """Load a (possibly nested) MATLAB struct from HDF5 into a nested dict. - - Python port of GIAnT-MATLAB ``loadStructFromH5.m``. Honors the - ``/row_major`` layout flag (absent => column-major, so 2-D+ axes are - transposed to recover MATLAB/README orientation). - - Parameters - ---------- - path : str or Path - Path to the HDF5 file. - - Returns - ------- - dict - The nested struct as a dict of dicts / ndarrays / scalars. - """ - with h5py.File(path, "r") as f: - row_major = False - if "row_major" in f: - row_major = bool(np.asarray(f["row_major"][()]).reshape(-1)[0]) - return _read_h5_group(f, row_major) - - -def write_dict_to_h5group(grp: h5py.Group, d: dict) -> None: - """Write a (JSON-serializable) dict into an h5 group key-by-key. - - Nested dicts become subgroups; scalars/arrays become datasets; bools are - stored as uint8; empty/None values are skipped; anything unexpected is - stringified. - - Parameters - ---------- - grp : h5py.Group - Destination group. - d : dict - The dict to serialize. - """ - str_dt = h5py.string_dtype(encoding="utf-8") - for key, v in d.items(): - key = str(key) - if isinstance(v, dict): - write_dict_to_h5group(grp.create_group(key), v) - elif isinstance(v, bool): - grp.create_dataset(key, data=np.uint8(v)) - elif isinstance(v, str): - grp.create_dataset(key, data=v, dtype=str_dt) - elif isinstance(v, (int, float)): - grp.create_dataset(key, data=v) - elif isinstance(v, (list, tuple)): - arr = np.asarray(v) - if arr.size == 0: - continue - if arr.dtype.kind in ("U", "S", "O"): - grp.create_dataset( - key, - data=np.array([str(x) for x in v], dtype=object), - dtype=str_dt, - ) - else: - grp.create_dataset(key, data=arr) - elif v is None: - continue - else: - grp.create_dataset(key, data=str(v), dtype=str_dt) - - -def _reshape_1d(src: dict, key: str): - """Return ``src[key]`` flattened to 1-D, or ``None`` if missing.""" - v = src.get(key) - return None if v is None else np.asarray(v).reshape(-1) - - -def load_alignment_data_h5(path: Union[str, Path]) -> dict: - """Load a ``*_ALIGNMENTDATA.h5`` file (aData) into flat arrays/scalars. - - Mirrors the aData struct written by BandRegistration.m. Motion estimates - are stored as column vectors and online-motion shifts live under the nested - ``slap2`` group in the new format; everything is flattened to 1-D here. - - ``motionDSr``/``motionDSc``/``motionDSz`` are negated relative to the - on-disk values. The on-disk convention (shared by ``BandRegistration.m`` - and ``MultiRoiRegistration.m``) is the sample's physical displacement, - i.e. ``reference_position = canonical_position - motionDS_saved``. Every - bandsilo consumer instead writes ``reference_position = - canonical_position + motion``, so the sign is flipped once here to keep - that arithmetic correct everywhere downstream. - - Parameters - ---------- - path : str or Path - Path to the ``*_ALIGNMENTDATA.h5`` file. - - Returns - ------- - dict - Keys ``DSframes``, ``motionDSr/c/z`` (sign-flipped from the on-disk - values, see above), ``onlineYshift/Xshift/Zshift``, ``numChannels``, - ``alignHz``. - """ - s = load_struct_from_h5(path) - slap2 = s.get("slap2", {}) or {} - # On disk, motionDSr/c/z is the sample displacement: BandRegistration.m - # searches a lookup table whose entry at offset `off` is - # ref(canonical + off), finds `motionDS_internal = off`, then saves - # `-motionDS_internal`. So `reference = canonical - motionDS_saved`. - # bandsilo consistently maps canonical superpixel coordinates into the - # reference stack as `canonical + motion` (see gui.py `sp_rows` and - # background.py `s_r`/`s_c`), which needs `motionDS_internal`. Negate - # back here, once, so that downstream arithmetic is correct. - motion_ds_r = _reshape_1d(s, "motionDSr") - motion_ds_c = _reshape_1d(s, "motionDSc") - motion_ds_z = _reshape_1d(s, "motionDSz") - return { - "DSframes": _reshape_1d(s, "DSframes"), - "motionDSr": None if motion_ds_r is None else -motion_ds_r, - "motionDSc": None if motion_ds_c is None else -motion_ds_c, - "motionDSz": None if motion_ds_z is None else -motion_ds_z, - "onlineYshift": _reshape_1d(slap2, "onlineMotionYshift"), - "onlineXshift": _reshape_1d(slap2, "onlineMotionXshift"), - "onlineZshift": _reshape_1d(slap2, "onlineMotionZshift"), - "numChannels": int(np.asarray(s["numChannels"]).reshape(-1)[0]), - "alignHz": float(np.asarray(s["alignHz"]).reshape(-1)[0]), - } - - -def _resolve_fn_adata(fn_adata: np.ndarray, moco_save_dr: str) -> np.ndarray: - """Resolve per-trial aData basenames to absolute paths under moco dir. - - Parameters - ---------- - fn_adata : ndarray of object, shape (n_dmds, n_trials) - Alignment-data basenames (may contain ``''``/``None``). - moco_save_dr : str - ``/motion_correction`` directory. - - Returns - ------- - ndarray of object, shape (n_dmds, n_trials) - Absolute paths (``''`` where the entry was empty). - """ - n_dmds, n_trials = fn_adata.shape - out = np.empty((n_dmds, n_trials), dtype=object) - for d in range(n_dmds): - for t in range(n_trials): - entry = fn_adata[d, t] - if entry in (None, ""): - out[d, t] = "" - else: - out[d, t] = os.path.join(moco_save_dr, str(entry)) - return out - - -def load_trial_table(path: Union[str, Path]) -> dict: - """Resolve a registered SLAP2 band ``trial_table.h5`` to a runtime view. - - Faithful load is delegated to :meth:`giant_python.models.TrialTable\ -.from_h5` (which mirrors the README schema exactly); this function performs - only the *structural* resolution the band backend needs: it hoists the - per-trial arrays out of ``slap2_info`` / ``motion_correction``, resolves - ``fn_adata`` basenames to absolute paths under - ``savedr/motion_correction``, and derives the result subdirectories and - ``(n_dmds, n_trials)``. The disk-dependent pieces (kept-trial mask, - per-DMD ``alignHz``, channel count) are computed separately by - :func:`compute_keep_trials` / :func:`read_align_info`. - - Parameters - ---------- - path : str or Path - Path to the ``trial_table.h5`` file. - - Returns - ------- - dict - Flat runtime view with keys ``datadr``, ``savedr``, ``moco_save_dr``, - ``annotation_save_dr``, ``src_extr_save_dr``, ``align_params``, - ``n_dmds``, ``n_trials``, ``filename``, ``first_line``, ``last_line``, - ``fn_adata`` (absolute paths), and ``ref_stack``. - - Raises - ------ - ValueError - If the file is not a registered SLAP2 band trial table (missing - ``slap2_info`` or ``motion_correction/fn_adata``). - """ - tt = TrialTable.from_h5(path) - slap2 = tt.slap2_info - motion_correction = tt.motion_correction or {} - if slap2 is None or "fn_adata" not in motion_correction: - raise ValueError( - "load_trial_table expects a registered SLAP2 band trial table " - "(slap2_info + motion_correction/fn_adata); " - f"got {str(path)!r}." - ) - - datadr = str(tt.datadr) if tt.datadr is not None else "" - result_dr = str(tt.savedr) if tt.savedr is not None else "" - moco_save_dr = os.path.join(result_dr, "motion_correction") - - align_params = { - key: to_serializable(val) - for key, val in ( - motion_correction.get("align_params", {}) or {} - ).items() - } - - filenames = np.atleast_2d(np.asarray(tt.filename, dtype=object)) - first_line = np.atleast_2d(np.asarray(slap2.first_line)) - last_line = np.atleast_2d(np.asarray(slap2.last_line)) - fn_adata = np.atleast_2d( - np.asarray(motion_correction["fn_adata"], dtype=object) - ) - - n_dmds, n_trials = filenames.shape - - return { - "datadr": datadr, - "savedr": result_dr, - "moco_save_dr": moco_save_dr, - "annotation_save_dr": os.path.join(result_dr, "annotations"), - "src_extr_save_dr": os.path.join(result_dr, "source_extraction"), - "align_params": align_params, - "n_dmds": int(n_dmds), - "n_trials": int(n_trials), - "filename": filenames, - "first_line": first_line, - "last_line": last_line, - "fn_adata": _resolve_fn_adata(fn_adata, moco_save_dr), - "ref_stack": slap2.ref_stack or {}, - } - - -def compute_keep_trials( - fn_adata_abs: np.ndarray, - filename: np.ndarray, - datadr: Union[str, Path], -) -> np.ndarray: - """Return the ``(n_dmds, n_trials)`` mask of trials whose files exist. - - A trial is kept only if its alignment-data file (absolute ``fn_adata`` - path) and its source file (``filename`` resolved under ``datadr``) both - exist on disk. Mirrors the ``verifyFiles`` / ``keepTrials`` logic in - ``extractSLAP2IntegrationSources.py``. - - Parameters - ---------- - fn_adata_abs : ndarray of object, shape (n_dmds, n_trials) - Absolute alignment-data paths (``''`` where absent). - filename : ndarray of object, shape (n_dmds, n_trials) - Source-file basenames, resolved under ``datadr``. - datadr : str or Path - Raw-data directory. - - Returns - ------- - ndarray of bool, shape (n_dmds, n_trials) - The kept-trial mask. - """ - fn_adata_abs = np.atleast_2d(np.asarray(fn_adata_abs, dtype=object)) - filename = np.atleast_2d(np.asarray(filename, dtype=object)) - n_dmds, n_trials = fn_adata_abs.shape - keep = np.ones((n_dmds, n_trials), dtype=bool) - for d in range(n_dmds): - for t in range(n_trials): - adata = fn_adata_abs[d, t] - src = filename[d, t] - src_path = os.path.join(str(datadr), str(src)) if src else "" - if ( - not adata - or not os.path.exists(str(adata)) - or not src - or not os.path.exists(src_path) - ): - keep[d, t] = False - return keep - - -def read_align_info( - fn_adata_abs: np.ndarray, - keep_trials: np.ndarray, - n_dmds: int, -) -> tuple: - """Read per-DMD ``alignHz`` and the channel count from alignment data. - - For each DMD, reads the first kept trial's ``_ALIGNMENTDATA.h5`` to recover - the alignment rate and acquisition channel count (which live in the - alignment files, not in ``trial_table.h5``). - - Parameters - ---------- - fn_adata_abs : ndarray of object, shape (n_dmds, n_trials) - Absolute alignment-data paths. - keep_trials : ndarray of bool, shape (n_dmds, n_trials) - Kept-trial mask from :func:`compute_keep_trials`. - n_dmds : int - Number of DMD paths. - - Returns - ------- - tuple - ``(align_hz, num_channels)`` -- a ``{"DMD{N}": alignHz}`` dict and the - channel count (``None`` if every DMD had no kept trials). - """ - fn_adata_abs = np.atleast_2d(np.asarray(fn_adata_abs, dtype=object)) - keep_trials = np.atleast_2d(np.asarray(keep_trials)) - align_hz = {} - num_channels = None - for d in range(n_dmds): - valid = np.flatnonzero(keep_trials[d]) - if valid.size == 0: - continue - a_data = load_alignment_data_h5(fn_adata_abs[d, valid[0]]) - align_hz[f"DMD{d + 1}"] = a_data["alignHz"] - num_channels = a_data["numChannels"] - return align_hz, num_channels diff --git a/src/giant_python/bandsilo/pipeline.py b/src/giant_python/bandsilo/pipeline.py deleted file mode 100644 index e9262af..0000000 --- a/src/giant_python/bandsilo/pipeline.py +++ /dev/null @@ -1,1034 +0,0 @@ -"""BandSILo driver: compose Phases 1-7 and write experiment_summary.h5. - -Ported from ``main()`` in ``extractSLAP2IntegrationSources.py`` (ref -1360-3108). :func:`extract_band_sources` runs the whole band-scan -source-extraction pipeline for one ``trial_table.h5``: load inputs -and geometry, read low-res trial data, estimate background/noise/rho, build the -activity image and detect peaks, localize sources with NMF, extract high-res -traces, assemble dF/F, and write the byte-compatible ``experiment_summary.h5`` -before returning a populated :class:`~giant_python.models.ExperimentSummary`. - -The orchestration driver itself touches ``slap2_utils`` reads, torch NMF, and -disk IO end-to-end, so it cannot be exercised without a full dataset and is -marked ``# pragma: no cover`` (its constituent Phase 1-7 kernels are each -unit-tested and cross-checked against the reference). The output assembly and -HDF5 writer are pure and fully tested here, since the on-disk schema is a -frozen contract. -""" - -from __future__ import annotations - -from dataclasses import dataclass -from pathlib import Path -from typing import List, Optional, Union - -import h5py -import numpy as np -import torch - -from ..models.experiment import ( - ExperimentSummary, - Source, - UserRoi, - Visualizations, -) -from ..models.params import SiloParams -from ..parallel import map_trials -from . import background as bg -from . import geometry as geo -from . import summary_images as si -from . import trial_data as td -from .annotate import ( - build_user_roi_geometry, - resolve_interactivity, - resolve_user_rois, -) -from .baseline import assemble_dff -from .hdf5 import ( - compute_keep_trials, - load_trial_table, - read_align_info, - to_serializable, - write_dict_to_h5group, -) -from .nmf import fit_sources -from .peaks import get_act_im_peaks -from .progress import log, progress -from .traces import get_high_res_traces - -_FRAME_INFO_KEYS = ( - "trial_num_frames", - "discard_frames", - "frame_line_idxs", - "offlineXshifts", - "offlineYshifts", - "offlineZshifts", - "onlineXshifts", - "onlineYshifts", - "onlineZshifts", -) - - -@dataclass -class PathResult: - """Per-DMD (``Path{N}``) results feeding the output writer. - - Bundles the localized sources, per-trial trace results, and summary images - for one imaging path, plus the geometry and windows needed to assemble the - ``experiment_summary.h5`` datasets. - - Attributes - ---------- - a : torch.Tensor - Spatial profiles ``(n_pixels, n_sources)``. - source_params : torch.Tensor - Source parameters ``(n_sources, 6)``. - source_snr : ndarray - Per-source low-resolution SNR ``(n_sources,)``. - act_im, mean_im, act_im_peaks : ndarray - Activity image, mean image, and raw activity-image peaks. - trace_results : list - Per-trial 8-tuples from - :func:`giant_python.bandsilo.traces.get_high_res_traces`. - z_depths : ndarray - Per-fast-Z reference-stack depth (``Path{N}/Z_depths``). - num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row, num_channels : int - Geometry / channel count. - denoise_window, baseline_window : int - Windows passed to :func:`giant_python.bandsilo.baseline.compute_f0`. - draw_user_rois : bool - Whether user ROIs were drawn (gates writing the user-ROI group). - soma_masks : list or None - Per-ROI boolean masks (``(fastz, rows, cols)`` each). - soma_labels : list or None - Per-ROI text labels. - yx_shape : tuple - ``(rows, cols)`` plane shape (for the empty-ROI mask shape). - """ - - a: torch.Tensor - source_params: torch.Tensor - source_snr: np.ndarray - act_im: np.ndarray - mean_im: np.ndarray - act_im_peaks: np.ndarray - trace_results: list - z_depths: np.ndarray - num_fast_zs: int - dmd_pixels_per_column: int - dmd_pixels_per_row: int - num_channels: int - denoise_window: int - baseline_window: int - draw_user_rois: bool = False - soma_masks: Optional[list] = None - soma_labels: Optional[list] = None - yx_shape: tuple = (0, 0) - - -def _with_nan_channel(arr: np.ndarray, num_channels: int) -> np.ndarray: - """Reshape ``(frames, sources)`` to ``(sources, channels, frames)``. - - Only the activity channel (index 0) is filled; the remaining channels are - NaN (reserved until multi-channel source extraction exists). - - Parameters - ---------- - arr : ndarray of shape (n_frames, n_sources) - Per-source trace. - num_channels : int - Number of acquisition channels. - - Returns - ------- - ndarray of shape (n_sources, num_channels, n_frames) - The reshaped trace with NaN in the non-activity channels. - """ - out = np.full( - (arr.shape[1], num_channels, arr.shape[0]), np.nan, dtype=arr.dtype - ) - out[:, 0, :] = arr.T - return out - - -def _assemble_user_rois(pr: PathResult) -> Optional[dict]: - """Assemble the ``user_rois`` datasets for one path, or ``None``. - - Parameters - ---------- - pr : PathResult - The path's results (uses ``draw_user_rois``/``soma_masks``/ - ``soma_labels`` and the per-trial ``F_soma`` from ``trace_results``). - - Returns - ------- - dict or None - ``{"mask", "labels", "F"}`` when user ROIs were drawn, else ``None``. - """ - if not pr.draw_user_rois or pr.soma_masks is None: - return None - if len(pr.soma_masks) > 0: - masks_by_roi = np.stack( - [np.asarray(m, dtype=bool) for m in pr.soma_masks], axis=0 - ) - else: - masks_by_roi = np.zeros((0, pr.num_fast_zs, *pr.yx_shape), dtype=bool) - f_soma_all = np.concatenate( - [r[7] for r in pr.trace_results], axis=0 - ).transpose(1, 2, 0) - return {"mask": masks_by_roi, "labels": pr.soma_labels, "F": f_soma_all} - - -def _dmd_user_rois(user_rois: Optional[dict], key: str) -> tuple: - """Return ``(soma_masks, soma_labels, soma_sps)`` for one DMD path. - - Slices a resolved session selection (from - :func:`giant_python.bandsilo.annotate.resolve_user_rois`) down to one DMD, - or the neutral ``(None, None, [])`` when user ROIs are disabled. - - Parameters - ---------- - user_rois : dict or None - The resolved session selection, or ``None`` when ``draw_user_rois`` - is unset. - key : str - The DMD key (``"DMD{N}"``). - - Returns - ------- - tuple - ``(soma_masks, soma_labels, soma_sps)`` -- the per-ROI masks and labels - (or ``None``) and the per-ROI superpixel index lists (``[]`` when - absent). - """ - if user_rois is None: - return None, None, [] - return ( - user_rois["user_roi_masks"].get(key), - user_rois["user_roi_labels"].get(key), - user_rois["user_roi_superpixels"].get(key, []), - ) - - -def assemble_path_outputs(pr: PathResult) -> dict: - """Assemble one path's HDF5-ready datasets from its results. - - Concatenates the per-trial traces, re-estimates ``F0`` and forms the - per-source dF/F, and shapes every ``Path{N}`` dataset (sources, frame info, - global fluorescence, visualizations, optional user ROIs). - - Parameters - ---------- - pr : PathResult - The path's results. - - Returns - ------- - dict - Keys mirror the ``experiment_summary.h5`` ``Path{N}`` datasets: - ``z_depths``, ``profiles``, ``coords``, ``dF_ls``, ``F0``, ``SNR``, - the frame-info arrays, ``global_f``, ``act_im``, ``mean_im``, - ``act_im_peaks``, and ``user_rois`` (or ``None``). - """ - results = pr.trace_results - d_f = np.concatenate([r[0] for r in results], axis=0) - f0_ls = np.concatenate([r[1] for r in results], axis=0) - f_full, f0, d_f, _ = assemble_dff( - d_f, f0_ls, pr.denoise_window, pr.baseline_window - ) - - profiles = pr.a.numpy().T.reshape( - -1, pr.num_fast_zs, pr.dmd_pixels_per_column, pr.dmd_pixels_per_row - ) - - return { - "z_depths": pr.z_depths, - "profiles": profiles, - "coords": pr.source_params[:, :3].numpy(), - "dF_ls": _with_nan_channel(d_f, pr.num_channels), - "F0": _with_nan_channel(f0, pr.num_channels), - "SNR": pr.source_snr.reshape(-1, 1), - "trial_num_frames": np.concatenate( - [[len(r[2])] for r in results] - ).reshape(-1, 1), - "discard_frames": np.any(np.isnan(f_full), axis=1).reshape(-1, 1), - "frame_line_idxs": np.concatenate( - [r[2] for r in results], axis=0 - ).reshape(-1, 1), - "offlineXshifts": np.concatenate( - [r[5][1] for r in results], axis=0 - ).reshape(-1, 1), - "offlineYshifts": np.concatenate( - [r[5][0] for r in results], axis=0 - ).reshape(-1, 1), - "offlineZshifts": np.concatenate( - [r[5][2] for r in results], axis=0 - ).reshape(-1, 1), - "onlineXshifts": np.concatenate( - [r[6][1] for r in results], axis=0 - ).reshape(-1, 1), - "onlineYshifts": np.concatenate( - [r[6][0] for r in results], axis=0 - ).reshape(-1, 1), - "onlineZshifts": np.concatenate( - [r[6][2] for r in results], axis=0 - ).reshape(-1, 1), - "global_f": np.concatenate([r[4] for r in results], axis=0).T, - "act_im": pr.act_im, - "mean_im": pr.mean_im, - "act_im_peaks": pr.act_im_peaks, - "user_rois": _assemble_user_rois(pr), - } - - -def _write_path_group(f: h5py.File, dmd_ix: int, assembled: dict) -> None: - """Write one ``Path{N}`` group from assembled outputs (overwriting).""" - group_name = f"Path{dmd_ix + 1}" - if group_name in f: - del f[group_name] - dmd_group = f.create_group(group_name) - dmd_group.create_dataset("Z_depths", data=assembled["z_depths"]) - - sources = dmd_group.create_group("sources") - spatial = sources.create_group("spatial") - spatial.create_dataset("profiles", data=assembled["profiles"]) - spatial.create_dataset("coords", data=assembled["coords"]) - temporal = sources.create_group("temporal") - temporal.create_dataset("dF_ls", data=assembled["dF_ls"]) - temporal.create_dataset("F0", data=assembled["F0"]) - temporal.create_dataset("SNR", data=assembled["SNR"]) - - frame_group = dmd_group.create_group("frame_info") - for key in _FRAME_INFO_KEYS: - frame_group.create_dataset(key, data=assembled[key]) - - global_group = dmd_group.create_group("global") - global_group.create_dataset("F", data=assembled["global_f"]) - - vis = dmd_group.create_group("visualizations") - vis.create_dataset("act_im", data=assembled["act_im"]) - vis.create_dataset("mean_im", data=assembled["mean_im"]) - vis.create_dataset("act_im_peaks", data=assembled["act_im_peaks"]) - - if assembled["user_rois"] is not None: - rois = assembled["user_rois"] - user_roi_group = dmd_group.create_group("user_rois") - user_roi_group.create_dataset("mask", data=rois["mask"]) - user_roi_group.create_dataset( - "labels", - data=np.array(rois["labels"], dtype=object).reshape(-1, 1), - dtype=h5py.string_dtype(encoding="utf-8"), - ) - user_roi_group.create_dataset("F", data=rois["F"]) - - -def write_experiment_summary( - output_path: Union[str, Path], - params_serializable: dict, - assembled_paths: List[dict], -) -> None: - """Write the byte-compatible ``experiment_summary.h5``. - - Writes the root ``row_major`` flag and ``params`` group once, then one - ``Path{N}`` group per DMD (overwriting any existing group of that name). - - Parameters - ---------- - output_path : str or Path - Destination ``experiment_summary.h5`` path. - params_serializable : dict - The analysis parameters (already run through - :func:`giant_python.bandsilo.hdf5.to_serializable`). - assembled_paths : list of dict - Per-path outputs from :func:`assemble_path_outputs`. - """ - with h5py.File(output_path, "a") as f: - if "row_major" not in f: - f["row_major"] = 1 - if "params" not in f: - write_dict_to_h5group( - f.create_group("params"), params_serializable - ) - for dmd_ix, assembled in enumerate(assembled_paths): - _write_path_group(f, dmd_ix, assembled) - - -def _sources_from_assembled(assembled: dict) -> List[Source]: - """Build the per-source :class:`Source` list for one path.""" - profiles = assembled["profiles"] - coords = assembled["coords"] - d_f = assembled["dF_ls"] - f0 = assembled["F0"] - snr = assembled["SNR"] - return [ - Source( - profile=profiles[j], - coords=coords[j], - df_ls=d_f[j], - f0=f0[j], - snr=float(snr[j, 0]), - ) - for j in range(profiles.shape[0]) - ] - - -def _user_rois_from_assembled(assembled: dict) -> List[UserRoi]: - """Build the per-ROI :class:`UserRoi` list for one path.""" - rois = assembled["user_rois"] - if rois is None: - return [] - labels = rois["labels"] - masks = rois["mask"] - f_soma = rois["F"] - return [ - UserRoi( - label=str(labels[k]), - mask=masks[k] if k < masks.shape[0] else None, - f=f_soma[k] if k < f_soma.shape[0] else None, - ) - for k in range(len(labels)) - ] - - -def build_experiment_summary( - params_serializable: dict, assembled_paths: List[dict] -) -> ExperimentSummary: - """Build the returned :class:`ExperimentSummary` from assembled outputs. - - Populates per-path sources and user ROIs; the single-view visualization / - z-depth / global-F / frame-info fields hold the last path's values (the - per-path structure lives in ``experiment_summary.h5``). - - Parameters - ---------- - params_serializable : dict - The analysis parameters. - assembled_paths : list of dict - Per-path outputs from :func:`assemble_path_outputs`. - - Returns - ------- - ExperimentSummary - The populated summary. - """ - summary = ExperimentSummary(params=params_serializable) - for assembled in assembled_paths: - summary.sources.append(_sources_from_assembled(assembled)) - summary.user_rois.append(_user_rois_from_assembled(assembled)) - summary.visualizations = Visualizations( - mean_im=assembled["mean_im"], - act_im=assembled["act_im"], - act_im_peaks=assembled["act_im_peaks"], - ) - summary.z_depths = assembled["z_depths"] - summary.global_f = assembled["global_f"] - summary.frame_info = {k: assembled[k] for k in _FRAME_INFO_KEYS} - return summary - - -def _resolve_params( - params_in, -) -> SiloParams: # pragma: no cover - trivial glue - """Coerce ``params_in`` (None / dict / SiloParams) to a SiloParams.""" - if params_in is None: - return SiloParams(scan_mode="band") - if isinstance(params_in, SiloParams): - return params_in - return SiloParams(**params_in) - - -def _resolve_session_user_rois( - result_dr: str, - trial_table: dict, - lookup: dict, - params: SiloParams, -) -> Optional[ - dict -]: # pragma: no cover - reference-stack/aData IO + optional GUI - """Resolve per-DMD user ROIs for the session, or ``None`` if disabled. - - When ``draw_user_rois`` is set, builds the per-DMD annotation geometry and - loads ``/annotations/annotations.h5`` (drawing it interactively, - or failing fast when headless, via - :func:`giant_python.bandsilo.annotate.resolve_user_rois`). Extraction - stays non-interactive: drawing only happens as the resolver's fallback. - - Parameters - ---------- - result_dr : str - Session results directory (holds the ``annotations`` subdir). - trial_table : dict - Normalized trial table. - lookup : dict - Band-registration lookup table. - params : SiloParams - Extraction parameters (``draw_user_rois``/``interactive``). - - Returns - ------- - dict or None - The resolved user-ROI selection, or ``None`` when ``draw_user_rois`` - is unset. - """ - if not params.draw_user_rois: - return None - user_roi_geo, ref_files = build_user_roi_geometry(trial_table, lookup) - return resolve_user_rois( - Path(result_dr) / "annotations", - trial_table["n_dmds"], - user_roi_geo, - interactive=resolve_interactivity(params.interactive), - ref_files=ref_files, - ) - - -def extract_band_sources( - path_to_trial_table: Union[str, Path], - params_in=None, -) -> ExperimentSummary: # pragma: no cover - full IO/slap2/torch orchestration - """Run the band source-extraction pipeline for one trial table. - - Loads the ``trial_table.h5`` and its geometry, then for each DMD path runs - the low-res read, background/rho estimation, activity-image peak detection, - NMF localization, and high-res trace extraction, writes - ``experiment_summary.h5``, and returns a populated - :class:`~giant_python.models.ExperimentSummary`. - - This driver reads SLAP2 files (``slap2_utils``), runs the torch NMF, and - performs disk IO, so it is not unit-tested (its Phase 1-7 kernels are). - - Parameters - ---------- - path_to_trial_table : str or Path - Path to the ``trial_table.h5`` produced by BandRegistration. - params_in : SiloParams or dict, optional - Parameter overrides. - - Returns - ------- - ExperimentSummary - The extracted sources and summary (also written to disk). - """ - params = _resolve_params(params_in) - log(f"Loading trial table {path_to_trial_table}", params.verbose) - trial_table = load_trial_table(path_to_trial_table) - result_dr = str(trial_table["savedr"]) - src_extr_dr = Path(result_dr) / "source_extraction" - src_extr_dr.mkdir(parents=True, exist_ok=True) - output_path = src_extr_dr / "experiment_summary.h5" - - lookup = geo.load_lookup_table( - Path(result_dr) / "motion_correction" / "bandRegLookupTable.h5", - trial_table["n_dmds"], - ) - psf = geo.load_psf(params.psf_dilation, trial_table["n_dmds"]) - - trial_table["keep_trials"] = compute_keep_trials( - trial_table["fn_adata"], trial_table["filename"], trial_table["datadr"] - ) - align_hz, num_channels = read_align_info( - trial_table["fn_adata"], - trial_table["keep_trials"], - trial_table["n_dmds"], - ) - trial_table["align_hz"] = align_hz - if params.num_channels is None: - params.num_channels = num_channels - if params.num_channels is None: - raise ValueError( - "num_channels could not be determined from the alignment data " - "(no kept trials?); set SiloParams.num_channels explicitly." - ) - - user_rois = _resolve_session_user_rois( - result_dr, trial_table, lookup, params - ) - - n_dmds = trial_table["n_dmds"] - log(f"Extracting sources for {n_dmds} DMD path(s)", params.verbose) - assembled_paths = [] - for dmd_ix in range(n_dmds): - assembled_paths.append( - assemble_path_outputs( - _process_dmd( - dmd_ix, trial_table, params, lookup, psf, user_rois - ) - ) - ) - - params_serializable = to_serializable(_params_dict(params)) - log(f"Writing {output_path}", params.verbose) - write_experiment_summary(output_path, params_serializable, assembled_paths) - return build_experiment_summary(params_serializable, assembled_paths) - - -def _params_dict(params: SiloParams) -> dict: # pragma: no cover - glue - """Return the on-disk params dict written to ``experiment_summary.h5``.""" - return { - "numChannels": params.num_channels, - "analyzeHz": params.analyze_hz, - "denoiseWindow_s": params.denoise_window_s, - "baselineWindow_s": params.baseline_window_s, - "backgroundInterpolation": params.background_interpolation, - "decayTau_s": params.decay_tau_s, - "dXY": params.d_xy, - "sparse_fac": params.sparse_fac, - "vif": params.vif, - "peakth": params.peakth, - "peak_buffer": params.peak_buffer, - "draw_user_rois": params.draw_user_rois, - "operator": params.operator, - } - - -def _process_dmd( - dmd_ix: int, - trial_table: dict, - params: SiloParams, - lookup: dict, - psf: dict, - user_rois: Optional[dict] = None, -) -> PathResult: # pragma: no cover - full per-DMD IO/slap2/torch compute - """Run Phases 3-7 for one DMD path and bundle the results. - - Reads the low-res trial data, estimates background/noise/rho, builds the - activity image and detects peaks, localizes sources with NMF, and extracts - high-res traces per trial (fanned out via - :func:`giant_python.parallel.map_trials`). When ``user_rois`` is provided, - this DMD's ROI masks/labels are attached and its per-ROI superpixels feed - the per-trial ``F_soma``. - """ - key = f"DMD{dmd_ix + 1}" - log(f"Processing {key}", params.verbose) - soma_masks, soma_labels, soma_sps = _dmd_user_rois(user_rois, key) - datadr = str(trial_table["datadr"]) - n_trials = trial_table["filename"].shape[1] - if params.max_trials is not None: - n_trials = min(n_trials, params.max_trials) - log(f"Debug: limiting to first {n_trials} trial(s)", params.verbose) - num_channels = params.num_channels - align_hz = trial_table["align_hz"][key] - keep_trials = trial_table["keep_trials"] - - ref_stack, _, _ = geo.load_reference_stack( - datadr, trial_table["ref_stack"], dmd_ix - ) - dmd_pixels_per_column = ref_stack.shape[2] - dmd_pixels_per_row = ref_stack.shape[3] - yx_shape = (dmd_pixels_per_column, dmd_pixels_per_row) - fastz_to_refz = lookup["fastZ2RefZ"][key] - num_fast_zs = fastz_to_refz.shape[0] - all_sp_ids = lookup["allSuperPixelIDs"][key] - num_super_pixels = all_sp_ids.shape[0] - n_pixels = num_fast_zs * dmd_pixels_per_column * dmd_pixels_per_row - - smi = geo.build_subsample_matrix_inds( - all_sp_ids, lookup["sparseMaskInds"][key] - ) - psf2d = psf[key] - sparse_h_inds, sparse_h_vals = geo.build_sparse_h( - smi, psf2d, dmd_pixels_per_column, dmd_pixels_per_row - ) - ref_d, ref_c, ref_r = geo.ref_pixs_to_drc( - smi[:, 0], dmd_pixels_per_column, dmd_pixels_per_row - ) - - def _read(trial_ix): - """Read one trial's low-res data (per-trial worker).""" - return td.read_band_trial_data( - trial_ix, - bool(keep_trials[dmd_ix, trial_ix]), - dmd_ix, - align_hz, - all_sp_ids, - datadr, - trial_table, - num_channels, - ) - - low = td.assemble_lowres_data( - map_trials( - _read, - range(n_trials), - params.max_workers, - desc=f"{key} reading trials" if params.verbose else None, - ), - num_channels, - ) - low_res_data_norm = low["lowResData"] / low["lowResDataCt"] - v_im = 1.0 / low["lowResDataCt"] - low2 = ( - low["lowResData2"] / low["lowResDataCt2"] - if num_channels >= 2 - else None - ) - - unique_motion, mot_inds = bg.bin_motion( - low["lowResMotionR"], low["lowResMotionC"], low["lowResMotionZ"] - ) - median_z = np.median(low["lowResMotionZ"]) - _, frames_to_keep = bg.select_motion_bins( - unique_motion, mot_inds, low["lowResMotionZ"] - ) - mean_im = si.compute_mean_image( - low_res_data_norm, - unique_motion, - mot_inds, - frames_to_keep, - ref_d, - ref_r, - ref_c, - num_fast_zs, - dmd_pixels_per_column, - dmd_pixels_per_row, - num_channels, - low2, - ) - umyx, mot_inds_yx = bg.bin_motion_yx( - low["lowResMotionR"], low["lowResMotionC"], frames_to_keep - ) - _, sel_pix_idxs = bg.build_selected_pixel_mask( - umyx, - ref_d, - ref_r, - ref_c, - num_fast_zs, - dmd_pixels_per_column, - dmd_pixels_per_row, - psf2d, - ) - pixel_coords = bg.pixel_coords_from_idxs( - sel_pix_idxs, dmd_pixels_per_column, dmd_pixels_per_row - ) - - background = _estimate_background( - low_res_data_norm, - umyx, - mot_inds_yx, - sel_pix_idxs, - ref_d, - ref_r, - ref_c, - num_fast_zs, - dmd_pixels_per_column, - dmd_pixels_per_row, - psf2d, - num_super_pixels, - align_hz, - params, - ) - data_std, _, _ = bg.fit_noise_variance_model( - low_res_data_norm, background, v_im, params.vif - ) - residual = bg.compute_residual(low_res_data_norm, background, data_std) - - act_im, act_im_peaks, a, source_params, source_snr = _localize( - residual, - umyx, - mot_inds_yx, - sel_pix_idxs, - pixel_coords, - n_pixels, - ref_d, - ref_r, - ref_c, - num_fast_zs, - dmd_pixels_per_column, - dmd_pixels_per_row, - psf2d, - num_super_pixels, - sparse_h_inds, - sparse_h_vals, - align_hz, - params, - ) - - def _traces(trial_ix): - """Extract one trial's high-res traces (per-trial worker).""" - return get_high_res_traces( - ( - trial_ix, - bool(keep_trials[dmd_ix, trial_ix]), - background[:, low["lowResTrialID"] == trial_ix], - ), - dmd_ix, - params.analyze_hz, - all_sp_ids, - datadr, - trial_table, - smi, - sparse_h_inds, - sparse_h_vals, - a, - umyx, - np.arange(umyx.shape[0]), - median_z, - psf2d, - num_super_pixels, - num_fast_zs, - dmd_pixels_per_column, - dmd_pixels_per_row, - num_channels, - soma_sps, - ) - - trace_results = map_trials( - _traces, - range(n_trials), - params.max_workers, - desc=f"{key} extracting traces" if params.verbose else None, - ) - trace_results = [r for r in trace_results if r[0].shape[0] > 0] - - return PathResult( - a=a, - source_params=source_params, - source_snr=source_snr, - act_im=act_im, - mean_im=mean_im, - act_im_peaks=act_im_peaks, - trace_results=trace_results, - z_depths=fastz_to_refz, - num_fast_zs=num_fast_zs, - dmd_pixels_per_column=dmd_pixels_per_column, - dmd_pixels_per_row=dmd_pixels_per_row, - num_channels=num_channels, - denoise_window=int( - np.ceil(params.denoise_window_s * params.analyze_hz) - ), - baseline_window=int( - np.ceil(params.baseline_window_s * params.analyze_hz) - ), - draw_user_rois=params.draw_user_rois, - soma_masks=soma_masks, - soma_labels=soma_labels, - yx_shape=yx_shape, - ) - - -def _estimate_background( - low_res_data_norm, - umyx, - mot_inds_yx, - sel_pix_idxs, - ref_d, - ref_r, - ref_c, - num_fast_zs, - dmd_pixels_per_column, - dmd_pixels_per_row, - psf2d, - num_super_pixels, - align_hz, - params, -): # pragma: no cover - Phase-4 background composition over real data - """Interpolated + rolling-baseline background on the superpixel grid.""" - log("Estimating background", params.verbose) - psf_tensor, psf_center, psf_exp, psf_center_exp = bg.expand_psf(psf2d) - interp_data = np.full( - (sel_pix_idxs.shape[0], low_res_data_norm.shape[1]), - np.nan, - dtype=np.float32, - ) - for z in progress( - range(num_fast_zs), desc="Interpolating planes", verbose=params.verbose - ): - z_idxs, sel_2d = bg.selected_pixels_2d_for_plane( - sel_pix_idxs, z, dmd_pixels_per_column, dmd_pixels_per_row - ) - if z_idxs.size == 0: - continue - ref_z = np.flatnonzero(ref_d == z) - interp_data[z_idxs], _, _ = bg.build_interp_data( - low_res_data_norm[ref_z], - ref_r[ref_z], - ref_c[ref_z], - sel_2d, - umyx, - mot_inds_yx, - method=params.background_interpolation, - ) - baseline_window = bg.baseline_window_frames( - align_hz, params.baseline_window_s - ) - interp_background = bg.compute_rolling_baseline( - interp_data, baseline_window - ) - return bg.assemble_background( - interp_background, - umyx, - mot_inds_yx, - sel_pix_idxs, - ref_d, - ref_r, - ref_c, - low_res_data_norm.shape[0], - low_res_data_norm.shape[1], - dmd_pixels_per_column, - dmd_pixels_per_row, - ) - - -def _localize( - residual, - umyx, - mot_inds_yx, - sel_pix_idxs, - pixel_coords, - n_pixels, - ref_d, - ref_r, - ref_c, - num_fast_zs, - dmd_pixels_per_column, - dmd_pixels_per_row, - psf2d, - num_super_pixels, - sparse_h_inds, - sparse_h_vals, - align_hz, - params, -): # pragma: no cover - Phase 4-6 rho/activity/NMF over real data - """Compute rho + activity image, detect peaks, and NMF-fit sources.""" - log("Computing activity image and localizing sources", params.verbose) - h_mots = bg.build_motion_h_matrices( - sparse_h_inds, - sparse_h_vals, - umyx, - sel_pix_idxs, - num_super_pixels, - dmd_pixels_per_row, - ) - # Build the activity-image D matrices from a narrow Gaussian center and a - # broad Gaussian surround instead of the PSF / expanded-PSF pair. - # ``compute_rho`` column-normalizes each and subtracts them, so the - # effective filter is H @ (G_center - G_surround). ``psf2d`` is still used - # there for the valid-column dilation/erosion mask. - # - # To restore the PSF-derived pair, delete the Gaussian kernels below, - # restore these two lines, - # (the ``shrink_psf`` call was itself a temporary hack and is optional), - # and swap the commented-out ``build_convolution_matrix`` calls back in - # inside the loop: - # psf2d = bg.shrink_psf(psf2d, scale_y=0.9, scale_x=0.75) - # psf_tensor, psf_center, psf_exp, psf_center_exp = bg.expand_psf(psf2d) - dog_center_sd = 0.9 - dog_surround_sd = 4.5 - center_kernel, center_kernel_ctr = bg.gaussian_kernel_2d(dog_center_sd) - surround_kernel, surround_kernel_ctr = bg.gaussian_kernel_2d( - dog_surround_sd - ) - # uniform_center_kernel = torch.ones((1, 1), dtype=torch.float32) - # uniform_center_kernel_ctr = (0, 0) - # uniform_surround_kernel = -torch.ones((3, 3), dtype=torch.float32) / 25.0 - # uniform_surround_kernel_ctr = (1, 1) - d_mats, d_mats_exp = [], [] - for z in progress( - range(num_fast_zs), - desc="Computing D matrices", - verbose=params.verbose, - ): - _, sel_2d = bg.selected_pixels_2d_for_plane( - sel_pix_idxs, z, dmd_pixels_per_column, dmd_pixels_per_row - ) - # d_mats.append( - # bg.build_convolution_matrix(sel_2d, psf_tensor, psf_center) - # ) - # d_mats_exp.append( - # bg.build_convolution_matrix(sel_2d, psf_exp, psf_center_exp) - # ) - d_mats.append( - bg.build_convolution_matrix( - sel_2d, center_kernel, center_kernel_ctr - ) - ) - d_mats_exp.append( - bg.build_convolution_matrix( - sel_2d, surround_kernel, surround_kernel_ctr - ) - ) - # d_mats.append( - # bg.build_convolution_matrix( - # sel_2d, uniform_center_kernel, uniform_center_kernel_ctr - # ) - # ) - # d_mats_exp.append( - # bg.build_convolution_matrix( - # sel_2d, - # uniform_surround_kernel, - # uniform_surround_kernel_ctr, - # ) - # ) - rho = bg.compute_rho( - residual, - mot_inds_yx, - umyx, - h_mots, - d_mats, - d_mats_exp, - sel_pix_idxs, - ref_d, - ref_r, - ref_c, - num_fast_zs, - dmd_pixels_per_column, - dmd_pixels_per_row, - psf2d, - verbose=params.verbose, - ) - nan_ct = bg.mask_high_nan_rho(rho) - rho = bg.smooth_rho( - rho, - bg.decay_kernel_1d(params.decay_tau_s, align_hz), - verbose=params.verbose, - ) - - act_im = si.accumulate_activity_image( - rho, - sel_pix_idxs, - num_fast_zs, - dmd_pixels_per_column, - dmd_pixels_per_row, - verbose=params.verbose, - ) - # Approximate the accumulated image's scan trajectory using the most - # populated retained YX motion bin. Dropped frames have bin index -1. - # Shift only the median geometry; rho/NMF still need the original centers. - motion_counts = np.bincount(mot_inds_yx[mot_inds_yx >= 0]) - dominant_motion = umyx[np.argmax(motion_counts)] - act_im = si.finalize_activity_image( - act_im, - sel_pix_idxs, - nan_ct, - ref_r=ref_r + int(dominant_motion[0]), - ref_c=ref_c + int(dominant_motion[1]), - ref_d=ref_d, - ) - - peak_th = params.peakth - source_seeds = get_act_im_peaks( - act_im, peak_th=peak_th, buffer_size=params.peak_buffer - ) - act_im_peaks = source_seeds.copy() - print(f"Number of detected sources: {act_im_peaks.shape[0]}") - - result = fit_sources( - source_seeds, - residual, - h_mots, - umyx, - mot_inds_yx, - sel_pix_idxs, - pixel_coords, - n_pixels, - params.d_xy, - params.sparse_fac, - verbose=params.verbose, - ) - return ( - act_im, - act_im_peaks, - result["A"], - result["source_params"], - result["source_snr"], - ) diff --git a/src/giant_python/cli.py b/src/giant_python/cli.py index 69ea23a..f71d94b 100644 --- a/src/giant_python/cli.py +++ b/src/giant_python/cli.py @@ -1,4 +1,4 @@ -"""Command-line interface: ``giant organize|register|annotate|extract``. +"""Command-line interface: ``giant annotate|extract``. Thin wrapper over the pipeline stages for batch / cluster use. Wired as the ``giant`` console script in pyproject.toml. The ``annotate`` subcommand is the @@ -6,9 +6,10 @@ annotation capsule): it writes ``annotations.h5``, which a later ``extract`` run consumes without opening a GUI. -The argument parser and the ``SiloParams`` builder are pure and unit-tested; -the individual command handlers drive heavy IO / GUI code and are excluded -from coverage. +CLI, standalone functions and Pipeline share the same workflow services. +Scientific defaults are unchanged; execution and annotation flags build +separate options. Heavy acquisition and GUI modules are imported only by +the selected backend when needed. """ from __future__ import annotations @@ -16,16 +17,26 @@ import argparse from typing import Optional, Sequence -from .models.params import SiloParams +from .models.params import ( + AnnotationOptions, + BandSiloParams, + ExecutionOptions, +) def _add_silo_options(parser: argparse.ArgumentParser) -> None: """Attach the shared source-extraction options to a subparser.""" + parser.add_argument( + "--microscope", + choices=("slap2",), + default="slap2", + help="Acquisition type (currently only SLAP2 band is implemented).", + ) parser.add_argument( "--scan-mode", - choices=("standard", "band"), + choices=("band",), default="band", - help="SLAP2 scan mode selecting the extraction backend.", + help="Supported scan mode.", ) parser.add_argument( "--draw-user-rois", @@ -49,7 +60,7 @@ def _add_silo_options(parser: argparse.ArgumentParser) -> None: ) parser.add_argument( "--operator", - default=SiloParams.operator, + default=AnnotationOptions.operator, help="Operator name recorded in the output metadata.", ) parser.add_argument( @@ -57,6 +68,18 @@ def _add_silo_options(parser: argparse.ArgumentParser) -> None: action="store_true", help="Print per-stage status messages and progress bars.", ) + parser.add_argument( + "--max-workers", + type=int, + default=ExecutionOptions.max_workers, + help="Positive worker count (default: 6).", + ) + parser.add_argument( + "--max-trials", + type=int, + default=None, + help="Positive debug trial limit per DMD (default: all trials).", + ) def build_parser() -> argparse.ArgumentParser: @@ -65,22 +88,13 @@ def build_parser() -> argparse.ArgumentParser: Returns ------- argparse.ArgumentParser - Parser with ``organize``, ``register``, ``annotate``, and ``extract`` - subcommands. Each subcommand stores its 0-based ``command`` name in the + Parser with ``annotate`` and ``extract`` + subcommands. Each subcommand stores its ``command`` name in the parsed namespace. """ parser = argparse.ArgumentParser(prog="giant") subparsers = parser.add_subparsers(dest="command") - organize = subparsers.add_parser( - "organize", help="Build the trial table from raw data." - ) - organize.add_argument("data_dir", help="Raw recording directory.") - organize.add_argument("save_dir", help="Results directory.") - - register = subparsers.add_parser("register", help="Run motion correction.") - register.add_argument("trial_table", help="Path to trial_table.h5.") - annotate = subparsers.add_parser( "annotate", help="Draw / save user ROIs (standalone step)." ) @@ -94,50 +108,55 @@ def build_parser() -> argparse.ArgumentParser: return parser -def _params_from_args(args: argparse.Namespace) -> SiloParams: - """Build a :class:`SiloParams` from parsed annotate/extract args.""" - return SiloParams( - scan_mode=args.scan_mode, - draw_user_rois=args.draw_user_rois, - interactive=args.interactive, - operator=args.operator, - verbose=args.verbose, +def _options_from_args( + args: argparse.Namespace, +) -> tuple[BandSiloParams, ExecutionOptions, AnnotationOptions]: + """Build independent science, execution and annotation configuration.""" + return ( + BandSiloParams(), + ExecutionOptions( + max_workers=args.max_workers, + verbose=args.verbose, + max_trials=args.max_trials, + ), + AnnotationOptions( + enabled=args.draw_user_rois or args.command == "annotate", + interactive=args.interactive, + operator=args.operator, + ), ) -def _cmd_organize(args: argparse.Namespace) -> int: # pragma: no cover - IO - """Handle ``giant organize`` (delegates to the trial-table builder).""" - from .pipeline import TrialTableBuilder - - TrialTableBuilder("slap2").run(args.data_dir, args.save_dir) - return 0 - - -def _cmd_register(args: argparse.Namespace) -> int: # pragma: no cover - IO - """Handle ``giant register`` (delegates to motion correction).""" - from .pipeline import MotionCorrector - - MotionCorrector.for_microscope("slap2").run(args.trial_table) - return 0 - - def _cmd_annotate( args: argparse.Namespace, -) -> int: # pragma: no cover - GUI/IO +) -> int: """Handle ``giant annotate`` (standalone ROI annotation).""" - from .bandsilo.annotate import annotate_band_rois + from .pipeline.annotate import annotate_rois - annotate_band_rois(args.trial_table, _params_from_args(args)) + params, execution, annotations = _options_from_args(args) + annotate_rois( + args.trial_table, + params, + microscope=args.microscope, + scan_mode=args.scan_mode, + execution=execution, + annotations=annotations, + ) return 0 -def _cmd_extract(args: argparse.Namespace) -> int: # pragma: no cover - IO +def _cmd_extract(args: argparse.Namespace) -> int: """Handle ``giant extract`` (source extraction).""" - from .pipeline import SourceExtractor + from .pipeline.extract import extract_sources - params = _params_from_args(args) - SourceExtractor.for_scan_mode(params.scan_mode, params).run( - args.trial_table + params, execution, annotations = _options_from_args(args) + extract_sources( + args.trial_table, + params, + microscope=args.microscope, + scan_mode=args.scan_mode, + execution=execution, + annotations=annotations, ) return 0 @@ -145,8 +164,7 @@ def _cmd_extract(args: argparse.Namespace) -> int: # pragma: no cover - IO def main(argv: Optional[Sequence[str]] = None) -> int: """Entry point for the ``giant`` console script. - Dispatches the ``organize``, ``register``, ``annotate``, and ``extract`` - subcommands to the corresponding pipeline stages. + Dispatches ``annotate`` and ``extract`` to their workflow services. Parameters ---------- @@ -157,19 +175,26 @@ def main(argv: Optional[Sequence[str]] = None) -> int: ------- int Process exit code (``2`` when no subcommand is given). + + Raises + ------ + SystemExit + Invalid options and unsupported workflows print an argparse error + and exit with code two, without claiming the operation succeeded. """ parser = build_parser() args = parser.parse_args(argv) handlers = { - "organize": _cmd_organize, - "register": _cmd_register, "annotate": _cmd_annotate, "extract": _cmd_extract, } if args.command is None: parser.print_help() return 2 - return handlers[args.command](args) + try: + return handlers[args.command](args) + except ValueError as exc: + parser.error(str(exc)) if __name__ == "__main__": # pragma: no cover diff --git a/src/giant_python/extraction/__init__.py b/src/giant_python/extraction/__init__.py new file mode 100644 index 0000000..2bcf7df --- /dev/null +++ b/src/giant_python/extraction/__init__.py @@ -0,0 +1 @@ +"""Source-extraction backends; import a backend explicitly when needed.""" diff --git a/src/giant_python/extraction/band/__init__.py b/src/giant_python/extraction/band/__init__.py new file mode 100644 index 0000000..3e3bb12 --- /dev/null +++ b/src/giant_python/extraction/band/__init__.py @@ -0,0 +1 @@ +"""Band-scan extraction kernels with no eager workflow or IO imports.""" diff --git a/src/giant_python/extraction/band/activity.py b/src/giant_python/extraction/band/activity.py new file mode 100644 index 0000000..959abfc --- /dev/null +++ b/src/giant_python/extraction/band/activity.py @@ -0,0 +1,333 @@ +"""Matched-filter activity, valid-column masking, and temporal smoothing.""" + +from __future__ import annotations + +from typing import List, Tuple + +import numpy as np +import scipy.ndimage as ndimage +import torch +from scipy import signal + +from ...progress import progress + + +def _plane_column_maps( + sel_pix_idxs: np.ndarray, num_fast_zs: int, plane_size: int +) -> Tuple[list, list, list]: + """Precompute per-z column masks, indices, and remaps into local space. + + Returns three lists (one entry per z): the boolean column mask, the column + indices of that plane's selected pixels, and a remap from global to local + (per-plane) column index (``-1`` outside the plane). + """ + sel_pix_z = sel_pix_idxs // plane_size + z_masks = [torch.from_numpy(sel_pix_z == z) for z in range(num_fast_zs)] + z_col_idxs = [torch.nonzero(m, as_tuple=False).squeeze(1) for m in z_masks] + z_remaps = [] + for z in range(num_fast_zs): + remap = torch.full((sel_pix_idxs.shape[0],), -1, dtype=torch.long) + if z_col_idxs[z].numel() > 0: + remap[z_col_idxs[z]] = torch.arange( + z_col_idxs[z].numel(), dtype=torch.long + ) + z_remaps.append(remap) + return z_masks, z_col_idxs, z_remaps + + +def _valid_sel_cols_for_motion( + motion_idx: int, + unique_motion_to_keep_yx: np.ndarray, + ref_d: np.ndarray, + ref_r: np.ndarray, + ref_c: np.ndarray, + sel_pix_idxs: np.ndarray, + num_fast_zs: int, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, + psf2d: np.ndarray, +) -> np.ndarray: + """Return a per-selected-pixel mask of columns valid for one motion bin. + + The valid region is the PSF-dilated reference support minus its + horizontally-eroded border (erosion width ``2 * psf_w - 1``), evaluated at + this bin's motion shift. + """ + plane_size = dmd_pixels_per_column * dmd_pixels_per_row + valid_pix_mask = np.zeros( + (num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row), dtype=bool + ) + valid_pix_mask[ + ref_d, + ref_r + int(unique_motion_to_keep_yx[motion_idx, 0]), + ref_c + int(unique_motion_to_keep_yx[motion_idx, 1]), + ] = True + valid_pix_mask = ndimage.binary_dilation( + valid_pix_mask, + structure=np.ones((1, psf2d.shape[0], psf2d.shape[1]), dtype=bool), + ) + valid_pix_mask = ndimage.binary_erosion( + valid_pix_mask, + # TODO: check if dimensions here should change + structure=np.ones( + (1, psf2d.shape[0], psf2d.shape[1] * 2 - 1), dtype=bool + ), + ) + valid_pix_idxs = np.flatnonzero(valid_pix_mask) + valid_lookup = np.zeros(num_fast_zs * plane_size, dtype=bool) + valid_lookup[valid_pix_idxs] = True + return valid_lookup[sel_pix_idxs] + + +def compute_rho( + residual: np.ndarray, + mot_inds_yx: np.ndarray, + unique_motion_to_keep_yx: np.ndarray, + h_mots: List[torch.Tensor], + d_mats: List[torch.Tensor], + d_mats_expanded: List[torch.Tensor], + sel_pix_idxs: np.ndarray, + ref_d: np.ndarray, + ref_r: np.ndarray, + ref_c: np.ndarray, + num_fast_zs: int, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, + psf2d: np.ndarray, + verbose: bool = False, +) -> np.ndarray: + """Compute the ``rho`` matched-filter response on the selected pixels. + + For each motion bin and z-plane, restricts ``H`` to that plane, forms the + normalized ``HD`` and ``HD_expanded`` projections, and projects the + residual onto ``HD - HD_expanded`` over the plane's valid columns. + + Parameters + ---------- + residual : ndarray of shape (n_superpixels, n_frames) + Noise-normalized residual from :func:`compute_residual`. + mot_inds_yx : ndarray of shape (n_frames,) + Per-frame motion-bin index (``-1`` for dropped frames). + unique_motion_to_keep_yx : ndarray of shape (n_yx_bins, 2) + Kept 2-D motion vectors. + h_mots : list of torch.Tensor + Per-motion sparse ``H`` from :func:`build_motion_h_matrices`. + d_mats, d_mats_expanded : list of torch.Tensor + Per-plane base/expanded convolution matrices from + :func:`build_convolution_matrix`. + sel_pix_idxs : ndarray of int + Sorted flat selected-pixel indices. + ref_d, ref_r, ref_c : ndarray + Per-superpixel reference depth/row/column indices. + num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row : int + Grid geometry. + psf2d : ndarray + The (cropped) PSF for this DMD. + verbose : bool + Show a per-motion-bin progress bar when set. + + Returns + ------- + ndarray of shape (n_sel, n_frames), float32 + The ``rho`` response (NaN where not computed). + """ + plane_size = dmd_pixels_per_column * dmd_pixels_per_row + z_masks, z_col_idxs, z_remaps = _plane_column_maps( + sel_pix_idxs, num_fast_zs, plane_size + ) + z_col_idxs_np = [zi.numpy() for zi in z_col_idxs] + + rho = np.full( + (len(sel_pix_idxs), residual.shape[1]), np.nan, dtype=np.float32 + ) + + for i in progress( + range(len(unique_motion_to_keep_yx)), + desc="Computing rho", + verbose=verbose, + ): + motion_frames = np.flatnonzero(mot_inds_yx == i) + if motion_frames.size == 0: + continue + + h_mot = h_mots[i].coalesce() + h_idxs = h_mot.indices() + h_vals = h_mot.values() + nrows, _ = h_mot.size() + residual_motion = residual[:, motion_frames].T + + valid_sel_cols = _valid_sel_cols_for_motion( + i, + unique_motion_to_keep_yx, + ref_d, + ref_r, + ref_c, + sel_pix_idxs, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + psf2d, + ) + + for z in range(num_fast_zs): + new_ncols = int(z_col_idxs[z].numel()) + if new_ncols == 0: + continue + + z_mask = z_masks[z] + keep_mask = z_mask[h_idxs[1]] + if keep_mask.sum().item() == 0: + # Unreachable defensive guard (mirrors the reference): a plane + # with selected pixels (new_ncols > 0) always has H columns in + # it, since both derive from the same in-plane PSF footprint. + continue # pragma: no cover + + remap = z_remaps[z] + new_rows = h_idxs[0, keep_mask] + new_cols = remap[h_idxs[1, keep_mask]] + new_idxs = torch.stack([new_rows, new_cols], dim=0) + new_vals = h_vals[keep_mask] + h_sub = torch.sparse_coo_tensor( + new_idxs, new_vals, (nrows, new_ncols), dtype=torch.float32 + ).coalesce() + + hd = torch.sparse.mm(h_sub, d_mats[z]) + hd = hd / torch.sum(hd, dim=0, keepdim=True) + # Center-surround matched filter. Both projections are + # column-normalized before subtracting, so ``hd_diff`` has + # zero-sum columns and is blind to spatially flat residual. + hd_expanded = torch.sparse.mm(h_sub, d_mats_expanded[z]) + hd_expanded = hd_expanded / torch.sum( + hd_expanded, dim=0, keepdim=True + ) + hd_diff = hd - hd_expanded + # TEMPORARY: whiten each filter column to unit L2 norm so every + # selected pixel's rho has the same noise variance. The L1 + # normalization above pins each column's *sum*, not its norm, so + # noise gain varies with where a column sits relative to the + # sparse superpixel reference-pixel lattice. Since the activity + # image accumulates rho^2 only at local maxima, that bias is + # amplified into a visible grid of bright reference pixels. + # Whitening also makes ``peak_th`` a true sigma threshold. + # To revert, delete the two lines below. + hd_norm = torch.linalg.norm(hd_diff, dim=0, keepdim=True) + hd_diff = hd_diff / hd_norm.clamp_min(1e-12) + + z_cols = z_col_idxs_np[z] + z_valid_mask = valid_sel_cols[z_cols] + if not np.any(z_valid_mask): + continue + z_valid_cols = z_cols[z_valid_mask] + local_valid_cols = remap[torch.from_numpy(z_valid_cols)].long() + + rho[np.ix_(z_valid_cols, motion_frames)] = ( + residual_motion @ hd_diff[:, local_valid_cols].numpy() + ).T + + return rho + + +def mask_high_nan_rho(rho: np.ndarray, thresh: float = 0.5) -> np.ndarray: + """NaN-out rows of ``rho`` that are mostly NaN and return the NaN fraction. + + ``rho`` is modified in place. Rows whose NaN fraction exceeds ``thresh`` + are set entirely to NaN. + + Parameters + ---------- + rho : ndarray of shape (n_sel, n_frames) + The rho response. + thresh : float + NaN-fraction threshold above which a row is dropped. + + Returns + ------- + ndarray of shape (n_sel,) + Per-row NaN fraction (before dropping). + """ + nan_ct = np.mean(np.isnan(rho), axis=1) + rho[nan_ct > thresh] = np.nan + return nan_ct + + +def decay_kernel_1d(decay_tau_s: float, align_hz: float) -> np.ndarray: + """Build the normalized causal exponential-decay smoothing kernel. + + Parameters + ---------- + decay_tau_s : float + Calcium-decay time constant, in seconds. + align_hz : float + Alignment/analysis rate, in Hz. + + Returns + ------- + ndarray + A 1-D kernel spanning ``3 * tau`` frames, summing to 1. + """ + decay_tau_frames = decay_tau_s * align_hz + k1d = np.exp( + np.linspace( + -np.ceil(decay_tau_frames * 3), + 0, + int(np.ceil(decay_tau_frames * 3) + 1), + ) + / decay_tau_frames + ) + return k1d / np.sum(k1d) + + +def smooth_rho( + rho: np.ndarray, k1d: np.ndarray, verbose: bool = False +) -> np.ndarray: + """NaN-aware temporal smoothing of ``rho`` with a 1-D decay kernel. + + Convolves along time only, dividing the smoothed values by the smoothed + valid-sample count and NaN-ing samples whose effective count drops below + ``0.75``. Rows are processed in chunks to bound temporary memory. + ``rho`` is modified in place and also returned. + + Parameters + ---------- + rho : ndarray of shape (n_sel, n_frames) + The rho response (NaN allowed). + k1d : ndarray + 1-D smoothing kernel from :func:`decay_kernel_1d`. + verbose : bool + Show a per-row-chunk progress bar when set. + + Returns + ------- + ndarray + The smoothed ``rho``. + """ + k2d = np.expand_dims(k1d, 0) + n_time = rho.shape[1] + bytes_per_row = max(1, n_time) * np.dtype(np.float32).itemsize * 3 + row_chunk = max(64, min(4096, int(200_000_000 // bytes_per_row))) + + for r0 in progress( + range(0, rho.shape[0], row_chunk), + desc="Smoothing rho", + verbose=verbose, + ): + r1 = min(r0 + row_chunk, rho.shape[0]) + rc = rho[r0:r1].copy() + row_has_data = np.any(np.isfinite(rc), axis=1) + if not np.any(row_has_data): + rho[r0:r1] = rc + continue + + rc_valid = rc[row_has_data] + rho_num = np.nan_to_num(rc_valid, nan=0.0) + rho_den = signal.convolve( + np.isfinite(rc_valid).astype(np.float32), k2d, mode="same" + ) + rho_num = signal.convolve(rho_num, k2d, mode="same") + valid_den = rho_den > 0.75 + np.divide(rho_num, rho_den, out=rho_num, where=valid_den) + rho_num[~valid_den] = np.nan + rc[row_has_data] = rho_num + rho[r0:r1] = rc + + return rho diff --git a/src/giant_python/extraction/band/annotation.py b/src/giant_python/extraction/band/annotation.py new file mode 100644 index 0000000..442115c --- /dev/null +++ b/src/giant_python/extraction/band/annotation.py @@ -0,0 +1,216 @@ +"""Standalone band annotation and shared load/draw/headless orchestration. + +Raw persistence lives in io.annotations, pure geometry in band.rois, and +toolkit code in gui.roi_editor. No toolkit is loaded on the headless path. +""" + +from __future__ import annotations + +import os +import sys +from pathlib import Path +from typing import Optional, Union + +import numpy as np + +from ...io.annotations import read_annotations_h5, save_annotations_h5 +from ...models.params import ( + AnnotationInput, + BandParamsInput, + ExecutionInput, + resolve_band_options, +) +from ...models.trial_table import TrialTable +from ...progress import log +from . import geometry as geo +from . import inputs +from .inputs import ( + compute_keep_trials, + load_alignment_data_h5, + load_trial_table, +) +from .rois import compute_user_roi_geometry, resolve_annotation_records + + +def first_valid_trial(keep_trials: np.ndarray, dmd_ix: int) -> int: + """Return this 0-based DMD's first kept trial, or raise ValueError.""" + indexes = np.flatnonzero(np.asarray(keep_trials)[dmd_ix]) + if indexes.size == 0: + raise ValueError(f"No valid trials for DMD {dmd_ix + 1}") + return int(indexes[0]) + + +def motion_median_from_adata(a_data: dict) -> tuple: + """Round internal band motion then take per-axis (row,column,z) medians.""" + return ( + int(np.nanmedian(np.round(a_data["motionDSr"]))), + int(np.nanmedian(np.round(a_data["motionDSc"]))), + int(np.nanmedian(np.round(a_data["motionDSz"]))), + ) + + +def _stdin_is_tty() -> bool: # pragma: no cover - runtime stdin + """Check for interactive stdin without importing a GUI toolkit.""" + return bool(getattr(sys.stdin, "isatty", lambda: False)()) + + +def resolve_interactivity(override: Optional[bool] = None) -> bool: + """Resolve override, then GIANT_HEADLESS, then terminal detection.""" + if override is not None: + return bool(override) + if os.environ.get("GIANT_HEADLESS"): + return False + return _stdin_is_tty() + + +def _empty_user_rois(n_dmds: int) -> dict: + """Create neutral per-path ROI selections without implying annotations.""" + keys = [f"DMD{d + 1}" for d in range(n_dmds)] + return { + "user_roi_masks": {k: [] for k in keys}, + "user_roi_superpixels": {k: [] for k in keys}, + "user_roi_labels": {k: [] for k in keys}, + "roi_records": {k: [] for k in keys}, + "annotated": False, + } + + +def load_annotations_h5( + annotations_path: str, n_dmds: int, user_roi_geo: dict +) -> tuple: + """Read and resolve band annotations; return the historical five-tuple. + + Missing/unreadable files and selections with no usable ROIs return + (False, {}, {}, {}, {}). Geometry validation is delegated to pure helpers. + """ + if not os.path.exists(annotations_path): + return False, {}, {}, {}, {} + try: + raw = read_annotations_h5(annotations_path) + except OSError as error: + print(f"Could not read {annotations_path}: {error}") + return False, {}, {}, {}, {} + return resolve_annotation_records(raw, n_dmds, user_roi_geo) + + +def resolve_user_rois( + annotation_dir: Union[str, Path], + n_dmds: int, + user_roi_geo: dict, + *, + interactive: bool, + ref_files: Optional[dict] = None, +) -> dict: + """Load usable annotations, or draw interactively or fail promptly.""" + path = os.path.join(str(annotation_dir), "annotations.h5") + skip, masks, superpixels, labels, records = load_annotations_h5( + path, n_dmds, user_roi_geo + ) + if skip: + return { + "user_roi_masks": masks, + "user_roi_superpixels": superpixels, + "user_roi_labels": labels, + "roi_records": records, + "annotated": True, + } + if not interactive: + raise RuntimeError( + "Annotations are enabled but no usable ROIs were found at " + f"{path!r}, and this process is not interactive. Run " + "`giant annotate ` first, or run in an " + "interactive session (or pass interactive=True) to draw now." + ) + return _draw_and_save_user_rois( + annotation_dir, n_dmds, user_roi_geo, ref_files + ) + + +def _draw_and_save_user_rois( + annotation_dir: Union[str, Path], + n_dmds: int, + user_roi_geo: dict, + ref_files: Optional[dict], +) -> dict: # pragma: no cover - GUI + """Launch the lazy ROI editor and save its records and masks.""" + from ...gui.roi_editor import run_user_roi_selection + + os.makedirs(str(annotation_dir), exist_ok=True) + selection = run_user_roi_selection(user_roi_geo, n_dmds) + save_annotations_h5( + str(annotation_dir), + selection["roi_records"], + selection["user_roi_masks"], + n_dmds, + ref_files=ref_files, + ) + selection["annotated"] = True + return selection + + +def build_user_roi_geometry(trial_table: dict, lookup: dict) -> tuple: + """Prepare per-DMD reference geometry and reference-file provenance.""" + user_roi_geo, ref_files = {}, {} + datadr = str(trial_table["datadr"]) + keep_trials = trial_table["keep_trials"] + for dmd_ix in range(int(trial_table["n_dmds"])): + key = f"DMD{dmd_ix + 1}" + ref_stack, _, ref_file = inputs.load_reference_stack( + datadr, trial_table["ref_stack"], dmd_ix + ) + smi = geo.build_subsample_matrix_inds( + lookup["allSuperPixelIDs"][key], lookup["sparseMaskInds"][key] + ) + fvt = first_valid_trial(keep_trials, dmd_ix) + a_data = load_alignment_data_h5(trial_table["fn_adata"][dmd_ix, fvt]) + user_roi_geo[key] = compute_user_roi_geometry( + ref_stack, + lookup["fastZ2RefZ"][key], + smi, + motion_median_from_adata(a_data), + ) + ref_files[key] = ref_file + return user_roi_geo, ref_files + + +def annotate_band_rois( + input: Union[str, Path, TrialTable], + params: BandParamsInput = None, + *, + execution: ExecutionInput = None, + annotations: AnnotationInput = None, +) -> str: # pragma: no cover - full reference IO and optional GUI + """Annotate a path or in-memory TrialTable; return annotations.h5's path. + + Existing usable annotations are loaded headlessly. No extraction or + source fitting occurs and in-memory trial tables are never reloaded. + Science, execution and annotation inputs are independently validated. + This explicit step requests annotation regardless of ``enabled``; + ``interactive`` still controls whether drawing is permitted. + """ + _, run, policy = resolve_band_options(params, execution, annotations) + log(f"Loading trial table {input}", run.verbose) + trial_table = load_trial_table(input) + trial_table["keep_trials"] = compute_keep_trials( + trial_table["fn_adata"], trial_table["filename"], trial_table["datadr"] + ) + annotation_dir = trial_table["annotation_save_dr"] + lookup = inputs.load_lookup_table( + Path(trial_table["moco_save_dr"]) / "bandRegLookupTable.h5", + trial_table["n_dmds"], + ) + log( + f"Loading reference geometry for {trial_table['n_dmds']} DMD path(s)", + run.verbose, + ) + user_roi_geo, ref_files = build_user_roi_geometry(trial_table, lookup) + resolve_user_rois( + annotation_dir, + trial_table["n_dmds"], + user_roi_geo, + interactive=resolve_interactivity(policy.interactive), + ref_files=ref_files, + ) + out_path = os.path.join(str(annotation_dir), "annotations.h5") + log(f"Annotations at {out_path}", run.verbose) + return out_path diff --git a/src/giant_python/extraction/band/background.py b/src/giant_python/extraction/band/background.py new file mode 100644 index 0000000..207009a --- /dev/null +++ b/src/giant_python/extraction/band/background.py @@ -0,0 +1,505 @@ +"""Band background interpolation, rolling baseline, and noise estimation.""" + +from __future__ import annotations + +from typing import Tuple + +import numpy as np +import pandas as pd +from scipy.interpolate import CubicSpline + +try: # pragma: no cover - exercised via the installed extra + import bottleneck as bn +except ImportError: # pragma: no cover + bn = None + + +def _collect_interp_columns( + cols, s_c, s_r, sel_idxs_by_col_all, sel_rows, r0, r1 +): + """Group a motion bin's reference pixels by column for interpolation. + + Columns with a single reference pixel become ``single_points`` (copied + verbatim); columns with several become sorted ``(rows, source-index)`` + entries for linear interpolation. + """ + rows_by_col = {} + idxs_by_col = {} + single_points = [] + for c in cols: + c_int = int(c) + if c_int not in sel_idxs_by_col_all: + continue + data_ix = np.flatnonzero(s_c == c) + if data_ix.size == 1: + rr = int(s_r[data_ix[0]]) + col_sel = sel_idxs_by_col_all[c_int] + rr_match = col_sel[sel_rows[col_sel] == rr] + if rr_match.size > 0 and (r0 <= rr <= r1): + single_points.append((rr_match, int(data_ix[0]))) + continue + rows = s_r[data_ix] + order = rows.argsort(kind="mergesort") + rows_by_col[c_int] = rows[order] + idxs_by_col[c_int] = data_ix[order] + return rows_by_col, idxs_by_col, single_points + + +# Preserve scientific parity with the reference interpolation branches. +def _interp_columns( # noqa: C901 + out, + rows_by_col, + idxs_by_col, + sel_idxs_by_col_all, + sel_rows, + data, + frames, + dtype, + method="linear", + spline_weight_cache=None, +): + """Interpolate each multi-point column onto its selected pixels. + + Selected rows that coincide with a reference row are copied exactly; those + strictly between reference rows use linear or natural-cubic interpolation; + those outside the sampled row range are left untouched (NaN). Cubic spline + weights depend only on row geometry, so they are computed once and applied + to all complete frames in one matrix multiplication. Linear interpolation + is retained for columns with fewer than three samples and frames containing + missing source values. + """ + if method not in {"linear", "cubic"}: + raise ValueError( + "interpolation method must be 'linear' or 'cubic', " + f"got {method!r}" + ) + for c_int, rows in rows_by_col.items(): + col_sel_pix_idxs = sel_idxs_by_col_all[c_int] + target_rows = sel_rows[col_sel_pix_idxs] + src_idx = idxs_by_col[c_int] + + pos = np.searchsorted(rows, target_rows, side="left") + in_bounds = pos < rows.size + exact = np.zeros_like(pos, dtype=bool) + exact[in_bounds] = rows[pos[in_bounds]] == target_rows[in_bounds] + + if np.any(exact): + exact_rows = np.flatnonzero(exact) + exact_src = src_idx[pos[exact]] + out[np.ix_(col_sel_pix_idxs[exact_rows], frames)] = data[ + exact_src + ][:, frames] + + lo = pos - 1 + hi = pos + interp_mask = (~exact) & (lo >= 0) & (hi < rows.size) + if not np.any(interp_mask): + continue + + interp_rows = np.flatnonzero(interp_mask) + lo_v = lo[interp_mask] + hi_v = hi[interp_mask] + row_lo = rows[lo_v].astype(dtype, copy=False) + row_hi = rows[hi_v].astype(dtype, copy=False) + denom = row_hi - row_lo + nonzero = denom != 0 + if not np.any(nonzero): + # Unreachable defensive guard (mirrors the reference): sorted + # distinct rows make searchsorted never straddle equal rows, so + # denom is always non-zero here. + continue # pragma: no cover + + interp_rows = interp_rows[nonzero] + lo_v = lo_v[nonzero] + hi_v = hi_v[nonzero] + row_lo = row_lo[nonzero] + row_hi = row_hi[nonzero] + alpha = ( + (target_rows[interp_rows].astype(dtype) - row_lo) + / (row_hi - row_lo) + )[:, None] + + target_indices = col_sel_pix_idxs[interp_rows] + linear_frame_mask = np.ones(frames.size, dtype=bool) + has_cubic_geometry = rows.size >= 3 and np.all(np.diff(rows) > 0) + if method == "cubic" and has_cubic_geometry: + source_values = data[src_idx][:, frames] + complete_frames = np.all(np.isfinite(source_values), axis=0) + if np.any(complete_frames): + source_rows = rows.astype(dtype, copy=False) + target_interp_rows = target_rows[interp_rows].astype( + dtype, copy=False + ) + origin = source_rows[0] + cache_key = ( + tuple(source_rows - origin), + tuple(target_interp_rows - origin), + np.dtype(dtype).str, + ) + spline_weights = ( + spline_weight_cache.get(cache_key) + if spline_weight_cache is not None + else None + ) + if spline_weights is None: + spline_weights = CubicSpline( + source_rows, + np.eye(rows.size, dtype=dtype), + axis=0, + bc_type="natural", + extrapolate=False, + )(target_interp_rows).astype(dtype, copy=False) + if spline_weight_cache is not None: + spline_weight_cache[cache_key] = spline_weights + cubic_values = ( + spline_weights @ source_values[:, complete_frames] + ) + out[np.ix_(target_indices, frames[complete_frames])] = ( + cubic_values + ) + linear_frame_mask = ~complete_frames + + if not np.any(linear_frame_mask): + continue + linear_frames = frames[linear_frame_mask] + vals_lo = data[src_idx[lo_v]][:, linear_frames] + vals_hi = data[src_idx[hi_v]][:, linear_frames] + linear_values = (1.0 - alpha) * vals_lo + alpha * vals_hi + out[np.ix_(target_indices, linear_frames)] = linear_values + + +def build_interp_data( + data: np.ndarray, + ref_r: np.ndarray, + ref_c: np.ndarray, + sel_pixels_2d: np.ndarray, + unique_motion_to_keep_yx: np.ndarray, + mot_inds_yx: np.ndarray, + height: int = 800, + width: int = 1280, + dtype=np.float32, + method: str = "cubic", +) -> Tuple[np.ndarray, Tuple[int, int], Tuple[int, int]]: + """Interpolate reference-pixel traces onto the selected-pixel grid. + + For each kept motion bin, the reference pixels (shifted by the bin's + motion) are linearly interpolated along rows, per column, onto the selected + pixels of the plane, filling one background estimate per selected pixel and + frame. Selected pixels outside the sampled row range remain NaN. + + Parameters + ---------- + data : ndarray of shape (n_ref, n_frames) + Reference-pixel traces for one plane. + ref_r, ref_c : ndarray + Reference-pixel row/column indices for this plane. + sel_pixels_2d : ndarray of shape (n_sel, 2) + Selected-pixel ``[row, col]`` coordinates for this plane. + unique_motion_to_keep_yx : ndarray of shape (n_yx_bins, 2) + Kept 2-D motion vectors. + mot_inds_yx : ndarray of shape (n_frames,) + Per-frame motion-bin index (``-1`` for dropped frames). + height, width : int + Sensor row/column extents used to window valid shifts. + dtype : numpy dtype + Working/output dtype. + method : {"linear", "cubic"} + Interpolation along each column (default ``"cubic"``). Cubic mode uses + a natural spline with precomputed geometry weights and falls back to + linear for columns with fewer than three samples or frames with missing + samples. + + Returns + ------- + out : ndarray of shape (n_sel, n_frames) + Interpolated background estimate (NaN where unsampled). + row_bounds : tuple of int + ``(r0, r1)`` inclusive row window. + col_bounds : tuple of int + ``(c0, c1)`` inclusive column window. + """ + ref_r = np.asarray(ref_r, dtype=np.int32) + ref_c = np.asarray(ref_c, dtype=np.int32) + data = np.asarray(data, dtype=dtype) + if method not in {"linear", "cubic"}: + raise ValueError( + "interpolation method must be 'linear' or 'cubic', " + f"got {method!r}" + ) + + r0 = int(max(0, ref_r.min()) + unique_motion_to_keep_yx[:, 0].min()) + r1 = int(min(height, ref_r.max()) + unique_motion_to_keep_yx[:, 0].max()) + c0 = int(max(0, ref_c.min()) + unique_motion_to_keep_yx[:, 1].min()) + c1 = int(min(width, ref_c.max()) + unique_motion_to_keep_yx[:, 1].max()) + + out = np.full((sel_pixels_2d.shape[0], data.shape[1]), np.nan, dtype=dtype) + + sel_cols = sel_pixels_2d[:, 1] + sel_rows = sel_pixels_2d[:, 0] + unique_sel_cols = np.unique(sel_cols) + sel_idxs_by_col_all = { + int(c): np.flatnonzero(sel_cols == c) for c in unique_sel_cols + } + + frames_by_motion = [ + np.flatnonzero(mot_inds_yx == idx) + for idx in range(len(unique_motion_to_keep_yx)) + ] + spline_weight_cache = {} if method == "cubic" else None + + for m_idx, frames in enumerate(frames_by_motion): + if frames.size == 0: + continue + + s_r = ref_r + int(unique_motion_to_keep_yx[m_idx, 0]) + s_c = ref_c + int(unique_motion_to_keep_yx[m_idx, 1]) + + in_win = (s_r >= r0) & (s_r <= r1) & (s_c >= c0) & (s_c <= c1) + if not np.any(in_win): + continue + cols = np.unique(s_c[in_win]) + + rows_by_col, idxs_by_col, single_points = _collect_interp_columns( + cols, s_c, s_r, sel_idxs_by_col_all, sel_rows, r0, r1 + ) + _interp_columns( + out, + rows_by_col, + idxs_by_col, + sel_idxs_by_col_all, + sel_rows, + data, + frames, + dtype, + method, + spline_weight_cache, + ) + for rr_match, src_pos in single_points: + out[np.ix_(rr_match, frames)] = data[src_pos, frames] + + return out, (r0, r1), (c0, c1) + + +def baseline_window_frames(align_hz: float, baseline_window_s: float) -> int: + """Return the rolling-baseline window length in frames. + + Parameters + ---------- + align_hz : float + Alignment/analysis rate, in Hz. + baseline_window_s : float + Baseline window, in seconds. + + Returns + ------- + int + ``int(align_hz * baseline_window_s)``. + """ + return int(align_hz * baseline_window_s) + + +def compute_rolling_baseline( + interp_data: np.ndarray, baseline_window: int +) -> np.ndarray: + """Compute a NaN-aware rolling-median baseline of the interpolated data. + + Each frame's baseline is the median of its centered window, ignoring NaNs, + so windows with no valid samples stay NaN and edge windows are truncated to + the available samples. This is the fast moving-median analogue of the + former moving-mean baseline; the median is more robust to transient + activity spikes when estimating the background. + + The heavy lifting uses :func:`bottleneck.move_median` (a C sliding-window + median whose cost is close to the moving mean) when available, falling back + to pandas' skip-list rolling median otherwise. ``bottleneck`` computes a + *trailing* window, so the data is right-padded with NaNs by ``window // 2`` + and the result is sliced back to center each window on its frame. + + Parameters + ---------- + interp_data : ndarray of shape (n_sel, n_frames) + Interpolated background estimate (from :func:`build_interp_data`). + baseline_window : int + Rolling window length, in frames. + + Returns + ------- + ndarray + The rolling-median background estimate (NaN where no valid samples). + """ + n_frames = interp_data.shape[1] + window = int(min(max(baseline_window, 1), n_frames)) + + if bn is not None: + pad = window // 2 + if pad: + padded = np.concatenate( + [ + interp_data, + np.full( + (interp_data.shape[0], pad), + np.nan, + dtype=interp_data.dtype, + ), + ], + axis=1, + ) + else: + padded = interp_data + trailing = bn.move_median(padded, window=window, min_count=1, axis=1) + background = trailing[:, pad : pad + n_frames] + return background.astype(interp_data.dtype, copy=False) + + # Fallback (no bottleneck): pandas rolls along the row axis, so transpose + # to (n_frames, n_sel), roll each column, and transpose back. + background = ( + pd.DataFrame(interp_data.T) + .rolling(window=window, center=True, min_periods=1) + .median() + .to_numpy() + .T + ) + return background.astype(interp_data.dtype, copy=False) + + +def assemble_background( + interp_data_background: np.ndarray, + unique_motion_to_keep_yx: np.ndarray, + mot_inds_yx: np.ndarray, + sel_pix_idxs: np.ndarray, + ref_d: np.ndarray, + ref_r: np.ndarray, + ref_c: np.ndarray, + n_super_pixels: int, + n_frames: int, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, +) -> np.ndarray: + """Map the selected-pixel background back onto superpixels per motion bin. + + For each kept motion bin and its frames, each superpixel's motion-shifted + reference pixel is looked up in ``sel_pix_idxs`` and its background trace + is copied onto that superpixel. + + Parameters + ---------- + interp_data_background : ndarray of shape (n_sel, n_frames) + Rolling-baseline background on the selected-pixel grid. + unique_motion_to_keep_yx : ndarray of shape (n_yx_bins, 2) + Kept 2-D motion vectors. + mot_inds_yx : ndarray of shape (n_frames,) + Per-frame motion-bin index (``-1`` for dropped frames). + sel_pix_idxs : ndarray of int + Sorted flat selected-pixel indices. + ref_d, ref_r, ref_c : ndarray + Per-superpixel reference depth/row/column indices. + n_super_pixels, n_frames : int + Output shape. + dmd_pixels_per_column, dmd_pixels_per_row : int + Grid geometry. + + Returns + ------- + ndarray of shape (n_super_pixels, n_frames), float32 + Per-superpixel background (NaN where unmapped). + """ + plane_size = dmd_pixels_per_column * dmd_pixels_per_row + background = np.full((n_super_pixels, n_frames), np.nan, dtype=np.float32) + + for motion_idx in range(len(unique_motion_to_keep_yx)): + motion_frames = np.flatnonzero(mot_inds_yx == motion_idx) + d_r = int(unique_motion_to_keep_yx[motion_idx, 0]) + d_c = int(unique_motion_to_keep_yx[motion_idx, 1]) + s_d = ref_d + s_r = ref_r + d_r + s_c = ref_c + d_c + + shifted_indices = s_d * plane_size + s_r * dmd_pixels_per_row + s_c + + bg_idxs = np.searchsorted(sel_pix_idxs, shifted_indices) + idxs_mask = bg_idxs < sel_pix_idxs.size + idxs_mask[idxs_mask] &= ( + sel_pix_idxs[bg_idxs[idxs_mask]] == shifted_indices[idxs_mask] + ) + + if np.any(idxs_mask): + valid_rows = np.flatnonzero(idxs_mask) + background[np.ix_(valid_rows, motion_frames)] = ( + interp_data_background[ + np.ix_(bg_idxs[valid_rows], motion_frames) + ] + ) + + return background + + +def fit_noise_variance_model( + low_res_data_norm: np.ndarray, + background: np.ndarray, + v_im: np.ndarray, + vif: float = 1.38, +) -> Tuple[np.ndarray, float, float]: + """Calibrate an affine noise-variance model and return per-sample std. + + Fits ``Var ~= Vk * (background * vIM) + Vb`` from the brightest pixels over + the first valid frames, then returns the predicted standard deviation for + every sample. + + Parameters + ---------- + low_res_data_norm : ndarray of shape (n_superpixels, n_frames) + Count-normalized channel-1 traces. + background : ndarray of shape (n_superpixels, n_frames) + Per-superpixel background estimate. + v_im : ndarray of shape (n_superpixels, n_frames) + Inverse count image (``1 / lowResDataCt``). + vif : float + Variance inflation factor. + + Returns + ------- + data_std : ndarray + Per-sample predicted standard deviation. + v_k : float + Fitted multiplicative variance coefficient. + v_b : float + Fitted additive variance floor. + """ + first_valid_frames = np.flatnonzero( + np.any(~np.isnan(background[:, :1000]), axis=0) + ) + var_im = np.nanvar(low_res_data_norm[:, first_valid_frames], axis=1) + v_b = np.nanpercentile(var_im, 5) * vif + var_pred = np.nanmean( + background[:, first_valid_frames], axis=1 + ) * np.nanmean(v_im[:, first_valid_frames], axis=1) + sel_bright = var_pred > np.nanpercentile(var_pred, 90) + v_k = np.nanpercentile( + (var_im[sel_bright] - (v_b / vif)) / var_pred[sel_bright], 10 + ) + data_std = np.sqrt(np.clip(v_k * background * v_im, 0, None) + v_b) + return data_std, v_k, v_b + + +def compute_residual( + low_res_data_norm: np.ndarray, + background: np.ndarray, + data_std: np.ndarray, +) -> np.ndarray: + """Return the background-subtracted, noise-normalized residual. + + Parameters + ---------- + low_res_data_norm : ndarray + Count-normalized traces. + background : ndarray + Per-superpixel background estimate. + data_std : ndarray + Per-sample predicted standard deviation. + + Returns + ------- + ndarray + ``(low_res_data_norm - background) / data_std``. + """ + return (low_res_data_norm - background) / data_std diff --git a/src/giant_python/extraction/band/geometry.py b/src/giant_python/extraction/band/geometry.py new file mode 100644 index 0000000..dd14cc4 --- /dev/null +++ b/src/giant_python/extraction/band/geometry.py @@ -0,0 +1,234 @@ +"""Pure DMD pixel geometry, selected-pixel support and PSF padding/cropping. + +Loading belongs to band.inputs; sparse projection to band.operators. +""" + +from __future__ import annotations + +from typing import Tuple + +import numpy as np +import scipy.ndimage as ndimage + + +def ref_pixs_to_drc( + ref_pixs: np.ndarray, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, +) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + """Map flat reference-pixel indices to DMD (depth, column, row) indices. + + Parameters + ---------- + ref_pixs : ndarray of int + Flat reference-pixel indices. + dmd_pixels_per_column, dmd_pixels_per_row : int + DMD geometry. + + Returns + ------- + ref_d, ref_c, ref_r : ndarray of int32 + Depth, column, and row indices. + """ + ref_pixs = np.asarray(ref_pixs, dtype=np.int64) + plane = int(dmd_pixels_per_column) * int(dmd_pixels_per_row) + npc = int(dmd_pixels_per_column) + ref_d = np.floor_divide(ref_pixs, plane).astype(np.int32) + ref_c = np.floor_divide( + ref_pixs - ref_d.astype(np.int64) * plane, npc + ).astype(np.int32) + ref_r = np.mod(ref_pixs, npc).astype(np.int32) + return ref_d, ref_c, ref_r + + +def build_subsample_matrix_inds( + all_super_pixel_ids: np.ndarray, + sparse_mask_inds: np.ndarray, +) -> np.ndarray: + """Build the superpixel -> reference-pixel index map for one DMD. + + For each superpixel, picks the element at index ``len(open_pixs) // 2`` + in sparse-mask order, for both odd and even counts; values are not sorted. + + Parameters + ---------- + all_super_pixel_ids : ndarray of shape (n_superpixels, 1) + Superpixel ids for this DMD. + sparse_mask_inds : ndarray of shape (N, 2) + ``[open_pixel (1-based), superpixel_id (1-based)]`` rows. + + Returns + ------- + ndarray of shape (n_superpixels, 2), int32 + ``[ref_pixel (0-based), superpixel_id (1-based)]`` per superpixel. + """ + num_super_pixels = all_super_pixel_ids.shape[0] + out = np.zeros((num_super_pixels, 2), dtype=np.int32) + for sp in range(num_super_pixels): + inds = np.where(sparse_mask_inds[:, 1] == sp + 1)[0] + open_pixs = sparse_mask_inds[inds, 0] - 1 + ref_pix = open_pixs[int(np.floor(len(open_pixs) / 2))] + out[sp, 0] = ref_pix + out[sp, 1] = sp + 1 + return out + + +def _pad_to(psf2d: np.ndarray, height: int, width: int) -> np.ndarray: + """Center-pad a 2-D PSF to ``(height, width)`` with its minimum value.""" + ph, pw = psf2d.shape + pad_r = (height - ph) // 2 + pad_c = (width - pw) // 2 + return np.pad( + psf2d, + ((pad_r, pad_r), (pad_c, pad_c)), + constant_values=np.min(psf2d), + ) + + +def build_combined_psf(psfs: list) -> np.ndarray: + """Stack per-DMD PSFs into a ``(n_dmds, H, W)`` array, center-padded.""" + height = max(p.shape[0] for p in psfs) + width = max(p.shape[1] for p in psfs) + combined = np.zeros((len(psfs), height, width), dtype=np.float32) + for i, p in enumerate(psfs): + combined[i] = _pad_to(p, height, width) + return combined + + +def threshold_and_crop_psf(psf2d: np.ndarray) -> np.ndarray: + """Zero values below ``max * exp(-3)`` and crop boundary zeros. + + Parameters + ---------- + psf2d : ndarray + A single-DMD PSF image. + + Returns + ------- + ndarray of float32 + The thresholded, tightly-cropped PSF. + """ + psf2d = psf2d.astype(np.float32, copy=True) + psf2d[psf2d < np.max(psf2d) * np.exp(-3)] = 0 + non_zero_rows = np.any(psf2d != 0, axis=1) + non_zero_cols = np.any(psf2d != 0, axis=0) + row_start, row_end = np.where(non_zero_rows)[0][[0, -1]] + col_start, col_end = np.where(non_zero_cols)[0][[0, -1]] + row_stop = row_end + 1 + col_stop = col_end + 1 + return psf2d[row_start:row_stop, col_start:col_stop] + + +def build_selected_pixel_mask( + unique_motion_to_keep_yx: np.ndarray, + ref_d: np.ndarray, + ref_r: np.ndarray, + ref_c: np.ndarray, + num_fast_zs: int, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, + psf2d: np.ndarray, +) -> Tuple[np.ndarray, np.ndarray]: + """Build the selected-pixel support mask over all kept motion shifts. + + Marks every reference pixel shifted by each kept 2-D motion bin, then + dilates in-plane by the PSF footprint. + + Parameters + ---------- + unique_motion_to_keep_yx : ndarray of shape (n_yx_bins, 2) + Kept 2-D motion vectors. + ref_d, ref_r, ref_c : ndarray + Per-superpixel reference depth/row/column indices. + num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row : int + Grid geometry. + psf2d : ndarray + The (cropped) PSF for this DMD; its shape sets the dilation footprint. + + Returns + ------- + sel_pix_mask : ndarray of bool + ``(num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row)`` support. + sel_pix_idxs : ndarray of int + Flattened indices of the selected pixels. + """ + sel_pix_mask = np.zeros( + (num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row), dtype=bool + ) + for i in range(len(unique_motion_to_keep_yx)): + sel_pix_mask[ + ref_d, + ref_r + int(unique_motion_to_keep_yx[i, 0]), + ref_c + int(unique_motion_to_keep_yx[i, 1]), + ] = True + sel_pix_mask = ndimage.binary_dilation( + sel_pix_mask, + structure=np.ones((1, psf2d.shape[0], psf2d.shape[1]), dtype=bool), + ) + sel_pix_idxs = np.flatnonzero(sel_pix_mask) + return sel_pix_mask, sel_pix_idxs + + +def pixel_coords_from_idxs( + sel_pix_idxs: np.ndarray, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, +) -> np.ndarray: + """Convert flat selected-pixel indices to ``[z, row, col]`` coordinates. + + Parameters + ---------- + sel_pix_idxs : ndarray of int + Flattened selected-pixel indices. + dmd_pixels_per_column, dmd_pixels_per_row : int + Grid geometry. + + Returns + ------- + ndarray of shape (n_sel, 3), int32 + ``[z, row, col]`` per selected pixel. + """ + plane_size = dmd_pixels_per_column * dmd_pixels_per_row + pixel_coords = np.empty((len(sel_pix_idxs), 3), dtype=np.int32) + pixel_coords[:, 0] = sel_pix_idxs // plane_size + remainder = sel_pix_idxs % plane_size + pixel_coords[:, 1] = remainder // dmd_pixels_per_row + pixel_coords[:, 2] = remainder % dmd_pixels_per_row + return pixel_coords + + +def selected_pixels_2d_for_plane( + sel_pix_idxs: np.ndarray, + plane_z: int, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, +) -> Tuple[np.ndarray, np.ndarray]: + """Return the in-plane ``[row, col]`` coords of one z-plane's pixels. + + Parameters + ---------- + sel_pix_idxs : ndarray of int + Flattened selected-pixel indices (all planes). + plane_z : int + The z-plane to select. + dmd_pixels_per_column, dmd_pixels_per_row : int + Grid geometry. + + Returns + ------- + z_idxs : ndarray + Positions (into ``sel_pix_idxs``) of pixels in ``plane_z``. + sel_pixels_2d : ndarray of shape (n_plane, 2) + ``[row, col]`` coordinates of those pixels. + """ + plane_size = dmd_pixels_per_column * dmd_pixels_per_row + sel_pix_z = sel_pix_idxs // plane_size + sel_pix_remainder = sel_pix_idxs % plane_size + z_idxs = np.flatnonzero(sel_pix_z == plane_z) + sel_pixels_2d = np.column_stack( + [ + sel_pix_remainder[z_idxs] // dmd_pixels_per_row, + sel_pix_remainder[z_idxs] % dmd_pixels_per_row, + ] + ) + return z_idxs, sel_pixels_2d diff --git a/src/giant_python/extraction/band/inputs.py b/src/giant_python/extraction/band/inputs.py new file mode 100644 index 0000000..90489a5 --- /dev/null +++ b/src/giant_python/extraction/band/inputs.py @@ -0,0 +1,337 @@ +"""Band runtime input preparation and displacement-to-offset normalization. + +Shared IO is faithful to saved displacement. Only this backend boundary +negates offline motion into the reference offsets used by band numerics. +Lookup/reference/PSF loading and asset discovery also belong at this boundary; +PSF padding and cropping delegate to pure geometry functions. +""" + +from __future__ import annotations + +import os +from importlib import resources +from pathlib import Path +from typing import Optional, Tuple, Union + +import numpy as np + +from ...io.hdf5 import load_struct_h5, to_serializable +from ...io.slap2 import load_alignment_data_h5 as read_alignment_data_h5 +from ...io.tiff import read_tiff +from ...models.trial_table import TrialTable +from .geometry import build_combined_psf, threshold_and_crop_psf + + +def load_lookup_table(path: Union[str, Path], n_dmds: int) -> dict: + """Load the band-registration lookup table (``bandRegLookupTable.h5``). + + Parameters + ---------- + path : str or Path + Path to the lookup-table HDF5 file. + n_dmds : int + Number of DMD paths. + + Returns + ------- + dict + ``{"allSuperPixelIDs": {...}, "sparseMaskInds": {...}, + "fastZ2RefZ": {...}}``, each an inner dict keyed ``DMD{N}``. + + Raises + ------ + ValueError + If a requested DMD has no Path/DMD lookup group. + """ + lt = load_struct_h5(path) + all_super_pixel_ids = {} + sparse_mask_inds = {} + fastz_to_refz = {} + for d in range(n_dmds): + p = lt.get(f"Path{d + 1}", lt.get(f"DMD{d + 1}")) + if p is None: + raise ValueError( + f"Missing lookup-table group Path{d + 1} or DMD{d + 1}" + ) + key = f"DMD{d + 1}" + all_super_pixel_ids[key] = ( + np.asarray(p["allSuperPixelIDs"]).reshape(-1, 1).astype(np.int32) + ) + sparse_mask_inds[key] = np.asarray(p["sparseMaskInds"]).astype( + np.int32 + ) + fastz_to_refz[key] = ( + np.asarray(p["fastZ2RefZ"]).reshape(-1, 1).astype(np.int32) + ) + return { + "allSuperPixelIDs": all_super_pixel_ids, + "sparseMaskInds": sparse_mask_inds, + "fastZ2RefZ": fastz_to_refz, + } + + +def _ref_stack_group(ref_stack: dict, dmd_ix: int) -> dict: + """Return a Path/DMD reference subgroup; raise ValueError if missing.""" + group = ref_stack.get( + f"Path{dmd_ix + 1}", ref_stack.get(f"DMD{dmd_ix + 1}") + ) + if group is None: + raise ValueError( + f"Missing reference-stack group Path{dmd_ix + 1} " + f"or DMD{dmd_ix + 1}" + ) + return group + + +def find_reference_file( + datadr: Union[str, Path], dmd_ix: int +) -> Optional[str]: + """Find the REFERENCE tif for a DMD under ``datadr`` (recursive glob). + + Parameters + ---------- + datadr : str or Path + Raw-data directory. + dmd_ix : int + 0-based DMD index. + + Returns + ------- + str or None + Path to the first matching REFERENCE tif, or ``None`` if none found. + """ + patterns = ( + f"**/*DMD{dmd_ix + 1}_CONFIG2-REFERENCE*", + f"**/*DMD{dmd_ix + 1}-REFERENCE*", + ) + for pattern in patterns: + matches = list(Path(datadr).glob(pattern)) + if matches: + return str(matches[0]) + return None + + +def load_reference_stack( + datadr: Union[str, Path], + ref_stack_meta: dict, + dmd_ix: int, +) -> Tuple[Optional[np.ndarray], np.ndarray, Optional[str]]: + """Load the band reference stack for one DMD. + + Reads the REFERENCE tif from ``datadr`` but takes the channel list from the + trial table's embedded ref_stack metadata. The raw counts are scaled by + ``1/100`` and reshaped to ``[channels, z, y, x]``. + + Parameters + ---------- + datadr : str or Path + Raw-data directory. + ref_stack_meta : dict + The trial table's ``ref_stack`` sub-struct. + dmd_ix : int + 0-based DMD index. + + Returns + ------- + ref_stack : ndarray or None + ``[channels, z, y, x]`` stack, or ``None`` if no REFERENCE file found. + channels : ndarray + Channel indices for this DMD. + ref_file : str or None + The REFERENCE file used, or ``None``. + + Raises + ------ + ValueError + If the requested DMD has no Path/DMD reference metadata group. + """ + grp = _ref_stack_group(ref_stack_meta, dmd_ix) + channels = np.asarray(grp["channels"]).reshape(-1) + num_channels = len(channels) + + ref_file = find_reference_file(datadr, dmd_ix) + if ref_file is None: + return None, channels, None + + ref = read_tiff(ref_file) / 100 + ref = ref.reshape(-1, num_channels, ref.shape[1], ref.shape[2]) + ref = ref.transpose(1, 0, 2, 3) + return ref, channels, ref_file + + +def default_psf(dilation: int) -> np.ndarray: + """Load a bundled default PSF template (``assets/psfs/dil-NN.tif``). + + Parameters + ---------- + dilation : int + Dilation size selecting ``dil-{dilation:02d}.tif`` (e.g. ``17``). + + Returns + ------- + ndarray of float32 + The PSF template image. + """ + name = f"dil-{int(dilation):02d}.tif" + resource = ( + resources.files("giant_python") + .joinpath("assets") + .joinpath("psfs") + .joinpath(name) + ) + with resources.as_file(resource) as p: + return read_tiff(str(p)).astype(np.float32) + + +def load_psf(dilation: int, n_dmds: int) -> dict: + """Load per-DMD PSFs from bundled assets, thresholded and cropped. + + Each DMD uses the same ``dil-NN.tif`` template from + ``assets/psfs/``, center-padded to a common size when shapes differ. + + Parameters + ---------- + dilation : int + Dilation size selecting the bundled PSF template. + n_dmds : int + Number of DMD paths. + + Returns + ------- + dict + ``{"DMD{N}": psf2d}`` thresholded/cropped PSFs. + """ + psfs = [default_psf(dilation) for _ in range(n_dmds)] + psf_combined = build_combined_psf(psfs) + return { + f"DMD{i + 1}": threshold_and_crop_psf(psf_combined[i]) + for i in range(n_dmds) + } + + +def load_alignment_data_h5(path: Union[str, Path]) -> dict: + """Read band alignment, negating saved offline shifts exactly once. + + Online shifts and frame indexes are unchanged. Negation allocates arrays + and does not modify values owned by the raw reader. + """ + data = read_alignment_data_h5(path) + return { + **data, + **{ + key: None if data[key] is None else -data[key] + for key in ("motionDSr", "motionDSc", "motionDSz") + }, + } + + +def _resolve_fn_adata(fn_adata: np.ndarray, moco_save_dr: str) -> np.ndarray: + """Resolve alignment basenames, preserving empty entries and array rank.""" + n_dmds, n_trials = fn_adata.shape + out = np.empty((n_dmds, n_trials), dtype=object) + for d in range(n_dmds): + for t in range(n_trials): + entry = fn_adata[d, t] + out[d, t] = ( + "" + if entry in (None, "") + else os.path.join(moco_save_dr, str(entry)) + ) + return out + + +def load_trial_table(path: Union[str, Path, TrialTable]) -> dict: + """Prepare a structural band view from a path OR an already loaded table. + + Never reload a TrialTable. This does not probe files or read alignments; + callers retain control of ``compute_keep_trials`` and ``read_align_info``. + The view carries the original model as ``trial_table`` and its provenance + as ``source_path``; all historical flattened runtime keys are retained. + """ + tt = path if isinstance(path, TrialTable) else TrialTable.from_h5(path) + slap2 = tt.slap2_info + motion_correction = tt.motion_correction or {} + if slap2 is None or "fn_adata" not in motion_correction: + raise ValueError( + "load_trial_table expects a registered SLAP2 band trial table " + "(slap2_info + motion_correction/fn_adata); " + f"got {str(path)!r}." + ) + datadr = str(tt.datadr) if tt.datadr is not None else "" + result_dr = str(tt.savedr) if tt.savedr is not None else "" + moco_save_dr = os.path.join(result_dr, "motion_correction") + align_params = { + key: to_serializable(val) + for key, val in ( + motion_correction.get("align_params", {}) or {} + ).items() + } + filenames = np.atleast_2d(np.asarray(tt.filename, dtype=object)) + first_line = np.atleast_2d(np.asarray(slap2.first_line)) + last_line = np.atleast_2d(np.asarray(slap2.last_line)) + fn_adata = np.atleast_2d( + np.asarray(motion_correction["fn_adata"], dtype=object) + ) + n_dmds, n_trials = filenames.shape + return { + "trial_table": tt, + "source_path": tt.source_path, + "datadr": datadr, + "savedr": result_dr, + "moco_save_dr": moco_save_dr, + "annotation_save_dr": os.path.join(result_dr, "annotations"), + "src_extr_save_dr": os.path.join(result_dr, "source_extraction"), + "align_params": align_params, + "n_dmds": int(n_dmds), + "n_trials": int(n_trials), + "filename": filenames, + "first_line": first_line, + "last_line": last_line, + "fn_adata": _resolve_fn_adata(fn_adata, moco_save_dr), + "ref_stack": slap2.ref_stack or {}, + } + + +def compute_keep_trials( + fn_adata_abs: np.ndarray, + filename: np.ndarray, + datadr: Union[str, Path], +) -> np.ndarray: + """Keep trials only when both alignment and raw source files exist.""" + fn_adata_abs = np.atleast_2d(np.asarray(fn_adata_abs, dtype=object)) + filename = np.atleast_2d(np.asarray(filename, dtype=object)) + n_dmds, n_trials = fn_adata_abs.shape + keep = np.ones((n_dmds, n_trials), dtype=bool) + for d in range(n_dmds): + for t in range(n_trials): + adata = fn_adata_abs[d, t] + src = filename[d, t] + src_path = os.path.join(str(datadr), str(src)) if src else "" + if ( + not adata + or not os.path.exists(str(adata)) + or not src + or not os.path.exists(src_path) + ): + keep[d, t] = False + return keep + + +def read_align_info( + fn_adata_abs: np.ndarray, + keep_trials: np.ndarray, + n_dmds: int, +) -> tuple: + """Read first-kept-trial alignHz per DMD and the last channel count.""" + fn_adata_abs = np.atleast_2d(np.asarray(fn_adata_abs, dtype=object)) + keep_trials = np.atleast_2d(np.asarray(keep_trials)) + align_hz = {} + num_channels = None + for d in range(n_dmds): + valid = np.flatnonzero(keep_trials[d]) + if valid.size == 0: + continue + a_data = load_alignment_data_h5(fn_adata_abs[d, valid[0]]) + align_hz[f"DMD{d + 1}"] = a_data["alignHz"] + num_channels = a_data["numChannels"] + return align_hz, num_channels diff --git a/src/giant_python/bandsilo/nmf.py b/src/giant_python/extraction/band/localization.py similarity index 96% rename from src/giant_python/bandsilo/nmf.py rename to src/giant_python/extraction/band/localization.py index dccc9c6..c23cafa 100644 --- a/src/giant_python/bandsilo/nmf.py +++ b/src/giant_python/extraction/band/localization.py @@ -2,8 +2,9 @@ Ported from the source-localization block of ``extractSLAP2IntegrationSources.py`` (ref 2519-2908). Given the peak seeds -from :mod:`giant_python.bandsilo.peaks` and the noise-normalized residual / -per-motion ``H`` matrices from :mod:`giant_python.bandsilo.background`, this +from :mod:`giant_python.numerics.peaks` and the noise-normalized residual / +per-motion ``H`` matrices from :mod:`giant_python.extraction.band.background` +and :mod:`giant_python.extraction.band.operators`, this module fits Gaussian spatial profiles (``A``) and per-motion temporal weights (``phi``) that reconstruct the residual in superpixel space. @@ -32,12 +33,12 @@ from __future__ import annotations -from typing import List +from typing import List, cast import numpy as np import torch -from .progress import progress +from ...progress import progress def sel_pix_gaussian_profile( @@ -272,7 +273,7 @@ def solve_phi_motion( Solves the regularized normal equations ``(XᵀX + 1e-10 I) phi = Xᵀ data`` and returns ``phiᵀ`` (frames x sources). Shared by the NMF fits here and the high-res trace solve in - :mod:`giant_python.bandsilo.traces`. + :mod:`giant_python.extraction.band.traces`. """ xtx = x.T @ x xtd = x.T @ data_motion @@ -365,10 +366,11 @@ def multiplicative_nmf( list of torch.Tensor Per-motion fitted superpixel spatial targets ``X_mots``. """ - x_mots: List[torch.Tensor] = [None] * n_motions + x_mots: List[torch.Tensor | None] = [None] * n_motions shuffled_indices = torch.randperm(n_motions) for idx in shuffled_indices: - i = idx.item() + # randperm uses an integer dtype, so item() returns a Python int. + i = cast(int, idx.item()) frames = _motion_frames(mot_inds_yx, i) x = torch.sparse.mm(h_mots[i], a[sel_pix_idxs, :]) @@ -400,7 +402,8 @@ def multiplicative_nmf( x = torch.where(norms > 0, x / norms, x) x_mots[i] = x - return x_mots + # randperm visits every slot exactly once, including the empty case. + return cast(List[torch.Tensor], x_mots) def fit_gaussian_profiles( @@ -487,7 +490,11 @@ def fit_gaussian_profiles( if epoch % n_motions == 0: shuffled_indices = torch.randperm(n_motions) - i = shuffled_indices[epoch % n_motions].item() + # Epoch zero initializes the permutation before any index is read. + i = cast( + int, + cast(torch.Tensor, shuffled_indices)[epoch % n_motions].item(), + ) x_step = torch.sparse.mm(h_mots[i], a_step) norms = torch.norm(x_step, dim=0, keepdim=True) @@ -509,7 +516,8 @@ def fit_gaussian_profiles( norms = torch.norm(x_step, dim=0, keepdim=True) x_step = torch.where(norms > 0, x_step / norms, x_step) total_loss += torch.sum((x_step - x_mots[i]) ** 2) - losses.append(total_loss.item()) + # This branch completes a nonempty motion round: loss is Tensor. + losses.append(cast(torch.Tensor, total_loss).item()) if ( epoch // n_motions > 0 and abs(losses[-1] - losses[-2]) < gd_tol @@ -663,7 +671,7 @@ def fit_sources( Parameters ---------- source_seeds : ndarray of shape (N, 3) - Peak seeds ``[z, mu_y, mu_x]`` from :mod:`giant_python.bandsilo.peaks`. + Peak seeds ``[z, mu_y, mu_x]`` from :mod:`giant_python.numerics.peaks`. residual : ndarray of shape (n_superpixels, n_frames) Noise-normalized residual to reconstruct. h_mots : list of torch.Tensor diff --git a/src/giant_python/extraction/band/motion_binning.py b/src/giant_python/extraction/band/motion_binning.py new file mode 100644 index 0000000..43d4055 --- /dev/null +++ b/src/giant_python/extraction/band/motion_binning.py @@ -0,0 +1,110 @@ +"""Rounded motion bins and low-resolution frame-selection policy.""" + +from __future__ import annotations + +from typing import Tuple + +import numpy as np + + +def bin_motion( + motion_r: np.ndarray, motion_c: np.ndarray, motion_z: np.ndarray +) -> Tuple[np.ndarray, np.ndarray]: + """Group frames into unique rounded (row, col, z) motion bins. + + Parameters + ---------- + motion_r, motion_c, motion_z : ndarray + Per-frame downsampled motion in rows, columns, and z. + + Returns + ------- + unique_motion : ndarray of shape (n_bins, 3) + The unique rounded ``[row, col, z]`` motion vectors. + mot_inds : ndarray of shape (n_frames,) + Index of each frame's motion bin into ``unique_motion``. + """ + unique_motion, mot_inds = np.unique( + np.round(np.stack((motion_r, motion_c, motion_z), axis=1)), + axis=0, + return_inverse=True, + ) + # np.unique's inverse is 1-D on numpy<2 and 2.1+, but 2-D on 2.0; flatten + # so downstream bincount/isin behave identically across versions. + return unique_motion, np.reshape(mot_inds, -1) + + +def select_motion_bins( + unique_motion: np.ndarray, + mot_inds: np.ndarray, + motion_z: np.ndarray, + z_thresh: float = 1.5, + min_frames: int = 100, +) -> Tuple[np.ndarray, np.ndarray]: + """Keep motion bins near the median z with enough frames. + + A bin is kept when its z is within ``z_thresh`` of the median z (over all + frames) and it contains more than ``min_frames`` frames. After filtering, + the remaining frames are treated as having no z motion. + + Parameters + ---------- + unique_motion : ndarray of shape (n_bins, 3) + Unique motion vectors from :func:`bin_motion`. + mot_inds : ndarray of shape (n_frames,) + Per-frame motion-bin indices. + motion_z : ndarray + Per-frame z motion (used for the median). + z_thresh : float + Maximum absolute z deviation from the median, in microns. + min_frames : int + Minimum frame count (strictly greater) for a bin to be kept. + + Returns + ------- + mot_inds_to_keep : ndarray + Indices of the kept bins into ``unique_motion``. + frames_to_keep : ndarray of bool + Per-frame mask of frames belonging to a kept bin. + """ + median_z = np.median(motion_z) + bin_counts = np.bincount(mot_inds, minlength=unique_motion.shape[0]) + keep_mask = (np.abs(unique_motion[:, 2] - median_z) <= z_thresh) & ( + bin_counts > min_frames + ) + mot_inds_to_keep = np.nonzero(keep_mask)[0] + frames_to_keep = np.isin(mot_inds, mot_inds_to_keep) + return mot_inds_to_keep, frames_to_keep + + +def bin_motion_yx( + motion_r: np.ndarray, + motion_c: np.ndarray, + frames_to_keep: np.ndarray, +) -> Tuple[np.ndarray, np.ndarray]: + """Bin kept frames into unique rounded (row, col) motion bins. + + Parameters + ---------- + motion_r, motion_c : ndarray + Per-frame downsampled row/column motion. + frames_to_keep : ndarray of bool + Frames to bin; dropped frames get index ``-1``. + + Returns + ------- + unique_motion_to_keep_yx : ndarray of shape (n_yx_bins, 2) + Unique rounded ``[row, col]`` motion vectors over the kept frames. + mot_inds_yx : ndarray of shape (n_frames,), int32 + Per-frame bin index into ``unique_motion_to_keep_yx``; ``-1`` for + frames not kept. + """ + mot_inds_yx = -1 * np.ones((frames_to_keep.shape[0],), dtype=np.int32) + stacked = np.round( + np.stack((motion_r, motion_c), axis=1)[frames_to_keep, :] + ) + unique_motion_to_keep_yx, inv = np.unique( + stacked, axis=0, return_inverse=True + ) + mot_inds_yx[frames_to_keep] = np.reshape(inv, -1) + return unique_motion_to_keep_yx, mot_inds_yx diff --git a/src/giant_python/extraction/band/operators.py b/src/giant_python/extraction/band/operators.py new file mode 100644 index 0000000..8161397 --- /dev/null +++ b/src/giant_python/extraction/band/operators.py @@ -0,0 +1,321 @@ +"""PSF kernels and sparse/dense band projection operators.""" + +from __future__ import annotations + +from typing import Any, List, Tuple, cast + +import numpy as np +import torch +from scipy.interpolate import RectBivariateSpline + +from .geometry import ref_pixs_to_drc + + +def expand_psf( + psf2d: np.ndarray, ex_fac: int = 2 +) -> Tuple[torch.Tensor, Tuple[int, int], torch.Tensor, Tuple[int, int]]: + """Normalize a PSF and build a center-aligned higher-resolution copy. + + The expanded PSF is an ``ex_fac``x up-sampled cubic-spline interpolation + that keeps the center aligned, used to form the ``HD - HD_expanded`` + matched filter in :func:`compute_rho`. + + Parameters + ---------- + psf2d : ndarray + The (cropped) PSF for this DMD. + ex_fac : int + Expansion factor along each axis. + + Returns + ------- + psf_tensor : torch.Tensor + Sum-normalized PSF. + psf_center : tuple of int + ``(row, col)`` center of ``psf_tensor``. + psf_tensor_expanded : torch.Tensor + Sum-normalized expanded PSF. + psf_center_expanded : tuple of int + ``(row, col)`` center of ``psf_tensor_expanded``. + """ + psf_arr = np.asarray(psf2d, dtype=np.float32) + psf_tensor = torch.from_numpy(psf_arr).float() + psf_tensor = psf_tensor / torch.sum(psf_tensor) + psf_shape = psf_arr.shape + psf_center = (psf_shape[0] // 2, psf_shape[1] // 2) + + expanded_center_y = (psf_shape[0] * ex_fac - 1) / 2 + expanded_center_x = (psf_shape[1] * ex_fac - 1) / 2 + orig_y = torch.linspace( + -expanded_center_y, expanded_center_y, psf_shape[0] + ) + orig_x = torch.linspace( + -expanded_center_x, expanded_center_x, psf_shape[1] + ) + expanded_y = torch.linspace( + -expanded_center_y, expanded_center_y, psf_shape[0] * ex_fac + ) + expanded_x = torch.linspace( + -expanded_center_x, expanded_center_x, psf_shape[1] * ex_fac + ) + interp_spline = RectBivariateSpline( + orig_y.numpy(), orig_x.numpy(), psf_tensor.numpy() + ) + psf_tensor_expanded = torch.from_numpy( + interp_spline(expanded_y.numpy(), expanded_x.numpy()) + ).float() + psf_tensor_expanded = psf_tensor_expanded / torch.sum(psf_tensor_expanded) + psf_shape_expanded = psf_tensor_expanded.shape + psf_center_expanded = ( + psf_shape_expanded[0] // 2, + psf_shape_expanded[1] // 2, + ) + return psf_tensor, psf_center, psf_tensor_expanded, psf_center_expanded + + +def shrink_psf( + psf2d: np.ndarray, scale_y: float = 1.0, scale_x: float = 0.75 +) -> np.ndarray: + """Spatially shrink a PSF on the same grid (TEMPORARY). + + Resamples the PSF on a grid whose coordinates are divided by the per-axis + scale so a ``scale < 1`` narrows the PSF's spatial footprint along that + axis while keeping the same array shape and center; the result is + re-normalized to sum to 1. Used only for the temporary activity-image + experiment. + + Parameters + ---------- + psf2d : ndarray + The (cropped) PSF for this DMD. + scale_y : float + Spatial shrink factor along the row (y) axis (``< 1`` shrinks, + ``1`` is a no-op). + scale_x : float + Spatial shrink factor along the column (x) axis (``< 1`` shrinks, + ``1`` is a no-op). + + Returns + ------- + ndarray of shape ``psf2d.shape``, float32 + The shrunk, sum-normalized PSF. + """ + psf_arr = np.asarray(psf2d, dtype=np.float32) + psf_shape = psf_arr.shape + center_y = (psf_shape[0] - 1) / 2 + center_x = (psf_shape[1] - 1) / 2 + orig_y = np.arange(psf_shape[0], dtype=np.float64) - center_y + orig_x = np.arange(psf_shape[1], dtype=np.float64) - center_x + interp_spline = RectBivariateSpline(orig_y, orig_x, psf_arr) + shrunk = interp_spline(orig_y / scale_y, orig_x / scale_x) + shrunk = np.clip(shrunk, 0.0, None).astype(np.float32) + total = shrunk.sum() + if total > 0: + shrunk = shrunk / total + return shrunk + + +def gaussian_kernel_2d( + sigma: float, truncate: float = 4.0 +) -> Tuple[torch.Tensor, Tuple[int, int]]: + """Build a sum-normalized isotropic 2-D Gaussian kernel (TEMPORARY). + + Used by the temporary difference-of-Gaussians activity-image experiment, + which replaces the PSF-derived ``D``/``D_expanded`` convolution matrices + with an isotropic center/surround pair. The subtraction itself still + happens in :func:`compute_rho`, after per-column normalization. + + Parameters + ---------- + sigma : float + Gaussian standard deviation, in pixels. + truncate : float + Kernel half-width, in standard deviations. Truncation lowers the + realized standard deviation slightly, so the default is generous. + + Returns + ------- + kernel : torch.Tensor + Sum-normalized ``(2 * radius + 1, 2 * radius + 1)`` kernel. + center : tuple of int + ``(row, col)`` center index of ``kernel``. + """ + radius = int(np.ceil(truncate * sigma)) + coords = np.arange(-radius, radius + 1, dtype=np.float64) + g_1d = np.exp(-(coords**2) / (2.0 * sigma**2)) + kernel = np.outer(g_1d, g_1d).astype(np.float32) + kernel /= kernel.sum() + return torch.from_numpy(kernel), (radius, radius) + + +def build_convolution_matrix( + sel_pixels_2d: np.ndarray, + psf_tensor: torch.Tensor, + psf_center: Tuple[int, int], +) -> torch.Tensor: + """Build a dense PSF-convolution matrix for one plane's selected pixels. + + Entry ``(t, s)`` is the PSF weight for the offset from source pixel ``s`` + to target pixel ``t`` (zero where the offset falls outside the PSF). Used + with both the base and expanded PSFs to form ``D``/``D_expanded``. + + Parameters + ---------- + sel_pixels_2d : ndarray of shape (n_plane, 2) + In-plane ``[row, col]`` coordinates. + psf_tensor : torch.Tensor + Sum-normalized PSF (base or expanded). + psf_center : tuple of int + ``(row, col)`` center of ``psf_tensor``. + + Returns + ------- + torch.Tensor + Dense ``(n_plane, n_plane)`` convolution matrix. + """ + n = sel_pixels_2d.shape[0] + if n == 0: + return torch.zeros((0, 0), dtype=torch.float32) + src_rows = sel_pixels_2d[:, 0][np.newaxis, :] + src_cols = sel_pixels_2d[:, 1][np.newaxis, :] + tgt_rows = sel_pixels_2d[:, 0][:, np.newaxis] + tgt_cols = sel_pixels_2d[:, 1][:, np.newaxis] + rel_rows = tgt_rows - src_rows + psf_center[0] + rel_cols = tgt_cols - src_cols + psf_center[1] + psf_np = psf_tensor.numpy() + psf_shape = psf_np.shape + valid_mask = ( + (rel_rows >= 0) + & (rel_rows < psf_shape[0]) + & (rel_cols >= 0) + & (rel_cols < psf_shape[1]) + ) + out = np.zeros((n, n), dtype=np.float32) + out[valid_mask] = psf_np[rel_rows[valid_mask], rel_cols[valid_mask]] + return torch.from_numpy(out) + + +def build_motion_h_matrices( + sparse_h_inds: np.ndarray, + sparse_h_vals: np.ndarray, + unique_motion_to_keep_yx: np.ndarray, + sel_pix_idxs: np.ndarray, + num_super_pixels: int, + dmd_pixels_per_row: int, +) -> List[torch.Tensor]: + """Build one sparse superpixel<-selected-pixel ``H`` per motion bin. + + Each motion bin shifts the base ``H`` columns (image pixels) by its 2-D + motion, then remaps them into selected-pixel column space. + + Parameters + ---------- + sparse_h_inds : ndarray of shape (2, nnz) + Base ``H`` COO row/column indices (from ``geometry.build_sparse_h``). + sparse_h_vals : ndarray of shape (nnz,) + Base ``H`` values. + unique_motion_to_keep_yx : ndarray of shape (n_yx_bins, 2) + Kept 2-D motion vectors. + sel_pix_idxs : ndarray of int + Sorted flat selected-pixel indices. + num_super_pixels : int + Number of superpixels (``H`` row count). + dmd_pixels_per_row : int + Grid geometry (column-shift stride). + + Returns + ------- + list of torch.Tensor + One sparse COO ``(num_super_pixels, n_sel)`` tensor per motion bin. + """ + n_mot = len(unique_motion_to_keep_yx) + h_mots: List[torch.Tensor | None] = [None] * n_mot + base_sparse_rows = sparse_h_inds[0] + base_sparse_cols = sparse_h_inds[1] + motion_shifts = unique_motion_to_keep_yx[:, 0].astype( + np.int64, copy=False + ) * dmd_pixels_per_row + unique_motion_to_keep_yx[:, 1].astype( + np.int64, copy=False + ) + shifted_inds = np.empty((2, base_sparse_cols.shape[0]), dtype=np.int64) + shifted_inds[0] = base_sparse_rows + for i, pix_shift in enumerate(motion_shifts): + shifted_cols = base_sparse_cols + pix_shift + shifted_inds[1] = np.searchsorted(sel_pix_idxs, shifted_cols) + # PyTorch accepts NumPy COO arrays despite its narrower stubs. + h_mots[i] = torch.sparse_coo_tensor( + cast(Any, shifted_inds), + cast(Any, sparse_h_vals), + (num_super_pixels, sel_pix_idxs.shape[0]), + dtype=torch.float32, + ) + # Every slot is populated by the corresponding motion shift above. + return cast(List[torch.Tensor], h_mots) + + +def build_sparse_h( + subsample_matrix_inds: np.ndarray, + psf2d: np.ndarray, + dmd_pixels_per_column: int, + dmd_pixels_per_row: int, +) -> Tuple[np.ndarray, np.ndarray]: + """Build the sparse ``H`` PSF-convolution matrix (COO index/value arrays). + + ``H`` projects image space (flattened ``z*(rows*cols) + row*cols + col``) + into superpixel space by convolving each superpixel's reference pixel with + the PSF. + + Parameters + ---------- + subsample_matrix_inds : ndarray of shape (n_superpixels, 2) + ``[ref_pixel (0-based), superpixel_id (1-based)]`` per superpixel. + psf2d : ndarray + The (thresholded/cropped) PSF for this DMD. + dmd_pixels_per_column, dmd_pixels_per_row : int + DMD plane geometry. + + Returns + ------- + sparse_h_inds : ndarray of shape (2, nnz), int32 + Row (superpixel) / column (image-pixel) indices of non-zero entries. + sparse_h_vals : ndarray of shape (nnz,), float32 + The corresponding PSF weights. + """ + ref_d, ref_c, ref_r = ref_pixs_to_drc( + subsample_matrix_inds[:, 0], dmd_pixels_per_column, dmd_pixels_per_row + ) + psf2d = np.asarray(psf2d, dtype=np.float32) + psf_h, psf_w = psf2d.shape + filter_size = psf_h * psf_w + plane_size = dmd_pixels_per_column * dmd_pixels_per_row + num_super_pixels = subsample_matrix_inds.shape[0] + + row_offsets = np.arange(psf_h, dtype=np.int32) - (psf_h // 2) + col_offsets = np.arange(psf_w, dtype=np.int32) - (psf_w // 2) + row_offsets_flat = np.broadcast_to( + row_offsets.reshape(-1, 1), (psf_h, psf_w) + ).ravel() + col_offsets_flat = np.broadcast_to( + col_offsets.reshape(1, -1), (psf_h, psf_w) + ).ravel() + psf_vals_flat = psf2d.ravel() + + sparse_h_inds = np.zeros( + (2, num_super_pixels * filter_size), dtype=np.int32 + ) + sparse_h_vals = np.zeros( + (num_super_pixels * filter_size,), dtype=np.float32 + ) + sparse_h_inds[0] = np.repeat(subsample_matrix_inds[:, 1] - 1, filter_size) + for sp in range(num_super_pixels): + start = sp * filter_size + end = (sp + 1) * filter_size + rows = ref_r[sp] + row_offsets_flat + cols = ref_c[sp] + col_offsets_flat + sparse_h_inds[1, start:end] = ( + ref_d[sp] * plane_size + rows * dmd_pixels_per_row + cols + ).astype(np.int32, copy=False) + sparse_h_vals[start:end] = psf_vals_flat + + non_zero_mask = sparse_h_vals != 0 + return sparse_h_inds[:, non_zero_mask], sparse_h_vals[non_zero_mask] diff --git a/src/giant_python/extraction/band/result_assembly.py b/src/giant_python/extraction/band/result_assembly.py new file mode 100644 index 0000000..09b46d7 --- /dev/null +++ b/src/giant_python/extraction/band/result_assembly.py @@ -0,0 +1,180 @@ +"""Canonical band numerical output assembly; persistence belongs to shared IO. + +Internal reference offsets become MATLAB displacement once here. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Optional + +import numpy as np +from torch import Tensor + +from ...models.experiment_summary import ( + FrameInfo, + PathSummary, + Source, + UserRoi, + Visualizations, +) +from ...numerics.baseline import assemble_dff +from .types import TrialTraceResult + + +@dataclass +class PathResult: + """One path's numerical outputs, geometry, and assembly window lengths. + + ``a`` is (pixels, sources), ``source_params`` is (sources, 6), and + trace_results contains named trial records. + Reference offsets use the INTERNAL band convention; + online shifts retain their saved displacement convention. + No tensor copies or numerical normalization occur in this container. + """ + + a: Tensor + source_params: Tensor + source_snr: np.ndarray + act_im: np.ndarray + mean_im: np.ndarray + act_im_peaks: np.ndarray + trace_results: list[TrialTraceResult] + z_depths: np.ndarray + num_fast_zs: int + dmd_pixels_per_column: int + dmd_pixels_per_row: int + num_channels: int + denoise_window: int + baseline_window: int + draw_user_rois: bool = False + soma_masks: Optional[list] = None + soma_labels: Optional[list] = None + yx_shape: tuple = (0, 0) + + +def _with_nan_channel(arr: np.ndarray, num_channels: int) -> np.ndarray: + """Map frames/sources to sources/channels/frames, filling channel 0.""" + out = np.full( + (arr.shape[1], num_channels, arr.shape[0]), np.nan, dtype=arr.dtype + ) + out[:, 0, :] = arr.T + return out + + +def _dmd_user_rois(user_rois: Optional[dict], key: str) -> tuple: + """Select one DMD's resolved ROI masks, labels and superpixels.""" + if user_rois is None: + return None, None, [] + return ( + user_rois["user_roi_masks"].get(key), + user_rois["user_roi_labels"].get(key), + user_rois["user_roi_superpixels"].get(key, []), + ) + + +def assemble_path_summary(pr: PathResult, path_index: int = 0) -> PathSummary: + """Concatenate trials and assemble dF/F with unchanged numerical ordering. + + Offline reference offsets are negated once into MATLAB displacement. + Online shifts, frame lines, input arrays, and interpolation are unchanged. + Populated model entries share array views; empty templates retain layout. + """ + results = pr.trace_results + d_f = np.concatenate([r.d_f for r in results], axis=0) + f0_ls = np.concatenate([r.f0_ls for r in results], axis=0) + f_full, f0, d_f, _ = assemble_dff( + d_f, f0_ls, pr.denoise_window, pr.baseline_window + ) + profiles = pr.a.numpy().T.reshape( + -1, pr.num_fast_zs, pr.dmd_pixels_per_column, pr.dmd_pixels_per_row + ) + coords = pr.source_params[:, :3].numpy() + df_channels = _with_nan_channel(d_f, pr.num_channels) + f0_channels = _with_nan_channel(f0, pr.num_channels) + snr = pr.source_snr.reshape(-1, 1) + sources = [ + Source( + profile=profiles[j], + coords=coords[j], + df_ls=df_channels[j], + f0=f0_channels[j], + snr=snr[j, 0], + ) + for j in range(profiles.shape[0]) + ] + frame_info = FrameInfo( + trial_num_frames=np.concatenate( + [[len(r.frame_line_idxs)] for r in results] + ).reshape(-1, 1), + discard_frames=np.any(np.isnan(f_full), axis=1).reshape(-1, 1), + frame_line_idxs=np.concatenate( + [r.frame_line_idxs for r in results], axis=0 + ).reshape(-1, 1), + offlineXshifts=-np.concatenate( + [r.reference_offsets[1] for r in results], axis=0 + ).reshape(-1, 1), + offlineYshifts=-np.concatenate( + [r.reference_offsets[0] for r in results], axis=0 + ).reshape(-1, 1), + offlineZshifts=-np.concatenate( + [r.reference_offsets[2] for r in results], axis=0 + ).reshape(-1, 1), + onlineXshifts=np.concatenate( + [r.online_shifts[1] for r in results], axis=0 + ).reshape(-1, 1), + onlineYshifts=np.concatenate( + [r.online_shifts[0] for r in results], axis=0 + ).reshape(-1, 1), + onlineZshifts=np.concatenate( + [r.online_shifts[2] for r in results], axis=0 + ).reshape(-1, 1), + ) + annotation_enabled = False + user_rois = [] + empty_roi_arrays = {} + if pr.draw_user_rois and pr.soma_masks is not None: + annotation_enabled = True + masks = ( + np.stack([np.asarray(m, dtype=bool) for m in pr.soma_masks]) + if len(pr.soma_masks) > 0 + else np.zeros((0, pr.num_fast_zs, *pr.yx_shape), dtype=bool) + ) + fluorescence = np.concatenate( + [r.user_roi_f for r in results], axis=0 + ).transpose(1, 2, 0) + labels = np.asarray(pr.soma_labels, dtype=object).reshape(-1) + user_rois = [ + UserRoi( + label=str(label), + mask=masks[j] if j < masks.shape[0] else None, + f=fluorescence[j] if j < fluorescence.shape[0] else None, + ) + for j, label in enumerate(labels) + ] + if not user_rois: + empty_roi_arrays = {"mask": masks, "F": fluorescence} + return PathSummary( + path_index=path_index, + sources=sources, + user_rois=user_rois, + visualizations=Visualizations( + act_im=pr.act_im, mean_im=pr.mean_im, act_im_peaks=pr.act_im_peaks + ), + frame_info=frame_info, + global_f=np.concatenate([r.global_f for r in results], axis=0).T, + z_depths=pr.z_depths, + annotation_enabled=annotation_enabled, + _empty_source_arrays=( + {} + if sources + else { + "profiles": profiles, + "coords": coords, + "dF_ls": df_channels, + "F0": f0_channels, + "SNR": snr, + } + ), + _empty_roi_arrays=empty_roi_arrays, + ) diff --git a/src/giant_python/extraction/band/rois.py b/src/giant_python/extraction/band/rois.py new file mode 100644 index 0000000..849e3fe --- /dev/null +++ b/src/giant_python/extraction/band/rois.py @@ -0,0 +1,158 @@ +"""Pure band ROI geometry, annotation validation, and superpixel mapping.""" + +import numpy as np + + +def user_roi_superpixel_lists_from_masks( + roi_masks: list, + sp_fastz: np.ndarray, + sp_rows: np.ndarray, + sp_cols: np.ndarray, +) -> list: + """Return indices of superpixels in each ROI, preserving overlap/order.""" + return [ + np.flatnonzero(np.asarray(mask)[sp_fastz, sp_rows, sp_cols]) + for mask in roi_masks + ] + + +def compute_user_roi_geometry( + ref: np.ndarray, + fastz_to_refz: np.ndarray, + subsample_matrix_inds: np.ndarray, + motion_median: tuple, +) -> dict: + """Prepare reference display and mask geometry with internal band offsets. + + Reference is (channels,z,Y,X); fast-Z lookup is 1-based. Reference pixel + decomposition retains the original column-major convention and indexes. + """ + avg_motion_r, avg_motion_c, avg_motion_z = motion_median + num_ref_z = ref.shape[1] + yx_shape = (ref.shape[2], ref.shape[3]) + plane = yx_shape[0] * yx_shape[1] + ch_means = [np.nanmean(ref[c]) for c in range(ref.shape[0])] + best_ch = int(np.argmax(ch_means)) if len(ch_means) > 0 else 0 + z_map = np.array(fastz_to_refz + avg_motion_z).reshape(-1) - 1 + num_fast_z = z_map.shape[0] + ref_pix = subsample_matrix_inds[:, 0] + sp_fastz = ref_pix // plane + sp_cols = avg_motion_c + (ref_pix - sp_fastz * plane) // yx_shape[0] + sp_rows = avg_motion_r + ref_pix % yx_shape[0] + sp_mask = np.zeros((num_fast_z, *yx_shape), dtype=bool) + valid = ( + (sp_rows >= 0) + & (sp_rows < yx_shape[0]) + & (sp_cols >= 0) + & (sp_cols < yx_shape[1]) + & (sp_fastz >= 0) + & (sp_fastz < num_fast_z) + ) + if np.any(valid): + sp_mask[sp_fastz[valid], sp_rows[valid], sp_cols[valid]] = True + return { + "ref": ref, + "num_ref_z": num_ref_z, + "yx_shape": yx_shape, + "best_ch": best_ch, + "z_map": z_map, + "num_fast_z": num_fast_z, + "sp_fastz": sp_fastz, + "sp_rows": sp_rows, + "sp_cols": sp_cols, + "sp_mask": sp_mask, + } + + +def _empty_path_selection(dmd_key: str, out: dict) -> None: + """Record a neutral selection for one invalid/missing path.""" + for values in out.values(): + values[dmd_key] = [] + + +def _read_roi_records(grp: dict, expected: tuple) -> tuple: + """Validate decoded ROI records against the current band mask geometry. + + Takes a plain decoded mapping, NOT an HDF5 handle. Missing masks become + empty masks, missing records are skipped, and bad shapes reject the path. + """ + n_rois = int(np.asarray(grp.get("n_rois", 0)).reshape(-1)[0]) + roi_masks, labels, recs = [], [], [] + for i in range(n_rois): + record = grp.get(f"roi_{i:03d}") + if record is None: + continue + label = str(record.get("label", f"ROI{i+1}")) + roi_type = str(record.get("type", "polygon")) + if "mask" in record: + mask = np.asarray(record["mask"]) + if mask.shape != expected: + return roi_masks, labels, recs, True + roi_masks.append(mask > 0) + else: + roi_masks.append(np.zeros(expected, dtype=bool)) + labels.append(label) + rec: dict[str, str | np.ndarray] = {"type": roi_type, "label": label} + if "position" in record: + rec["position"] = np.asarray(record["position"]) + recs.append(rec) + return roi_masks, labels, recs, False + + +def _load_one_path( + raw: dict, dmd_ix: int, user_roi_geo: dict, out: dict +) -> bool: + """Resolve one decoded path against current geometry, without disk IO.""" + key = f"DMD{dmd_ix + 1}" + path_key = f"Path{dmd_ix + 1}" + geo = user_roi_geo[key] + expected = (geo["num_fast_z"], geo["yx_shape"][0], geo["yx_shape"][1]) + group = raw.get(path_key) + if group is None: + _empty_path_selection(key, out) + return False + masks, labels, records, bad_shape = _read_roi_records(group, expected) + if bad_shape: + print( + f"annotations.h5: {path_key} ROI mask shape mismatch " + f"(expected {expected}); treating this path as no selection." + ) + _empty_path_selection(key, out) + return False + out["user_roi_masks"][key] = masks + out["user_roi_superpixels"][key] = user_roi_superpixel_lists_from_masks( + masks, geo["sp_fastz"], geo["sp_rows"], geo["sp_cols"] + ) + out["user_roi_labels"][key] = labels + out["roi_records"][key] = records + if len(records) > 0: + print( + f"Loaded {len(records)} ROI(s) for {path_key} from annotations.h5" + ) + return True + return False + + +def resolve_annotation_records( + raw: dict, n_dmds: int, user_roi_geo: dict +) -> tuple: + """Return selection status, masks, superpixels, labels, and ROI records.""" + out = { + "user_roi_masks": {}, + "user_roi_superpixels": {}, + "user_roi_labels": {}, + "roi_records": {}, + } + any_valid = False + for dmd_ix in range(n_dmds): + if _load_one_path(raw, dmd_ix, user_roi_geo, out): + any_valid = True + if not any_valid: + return False, {}, {}, {}, {} + return ( + True, + out["user_roi_masks"], + out["user_roi_superpixels"], + out["user_roi_labels"], + out["roi_records"], + ) diff --git a/src/giant_python/bandsilo/summary_images.py b/src/giant_python/extraction/band/summary_images.py similarity index 84% rename from src/giant_python/bandsilo/summary_images.py rename to src/giant_python/extraction/band/summary_images.py index 67eb4e0..610b45f 100644 --- a/src/giant_python/bandsilo/summary_images.py +++ b/src/giant_python/extraction/band/summary_images.py @@ -7,8 +7,8 @@ trace onto the reference-pixel grid, producing the ``mean_im`` visualization. * :func:`accumulate_activity_image` / :func:`finalize_activity_image` (ref 2358-2504) turn the smoothed ``rho`` from - :mod:`giant_python.bandsilo.background` into the ``act_im`` activity image - that Phase-5 peak detection localizes sources in. + :mod:`giant_python.extraction.band.activity` into the ``act_im`` activity + image that Phase-5 peak detection localizes sources in. The activity image is built by sweeping ``rho`` in temporal batches, finding spatio-temporal local maxima (a voxel greater than its 6 spatial neighbors in @@ -22,11 +22,14 @@ from __future__ import annotations +from typing import cast + import numpy as np import scipy.ndimage as ndimage +from numpy.typing import NDArray -from .progress import progress -from .trial_data import fast_dilation +from ...numerics.morphology import fast_dilation +from ...progress import progress def compute_mean_image( @@ -41,7 +44,7 @@ def compute_mean_image( dmd_pixels_per_column: int, dmd_pixels_per_row: int, num_channels: int, - low_res_data2_norm: np.ndarray = None, + low_res_data2_norm: np.ndarray | None = None, ) -> np.ndarray: """Scatter the kept-frame mean trace onto the reference pixel grid. @@ -55,7 +58,7 @@ def compute_mean_image( Count-normalized channel-1 low-res traces. unique_motion : ndarray of shape (n_bins, 3) Unique motion vectors from - :func:`giant_python.bandsilo.background.bin_motion`. + :func:`giant_python.extraction.band.motion_binning.bin_motion`. mot_inds : ndarray Per-frame motion-bin indices (used to pick the most common bin). frames_to_keep : ndarray of bool @@ -88,8 +91,9 @@ def compute_mean_image( low_res_data_norm[:, frames_to_keep], axis=1 ) if num_channels >= 2: + # The two-channel input contract requires the second-channel array. mean_im[1, ref_d, sel_r, sel_c] = np.nanmean( - low_res_data2_norm[:, frames_to_keep], axis=1 + cast(np.ndarray, low_res_data2_norm)[:, frames_to_keep], axis=1 ) return mean_im @@ -196,12 +200,12 @@ def accumulate_activity_image( nm.fill(True) time_indices = time_indices_pre[:curr_size] batch_vals = rho[:, padded_start:padded_end].T - br[ - time_indices, depth_indices, row_indices_roi, col_indices_roi - ] = batch_vals - nm[ - time_indices, depth_indices, row_indices_roi, col_indices_roi - ] = np.isnan(batch_vals) + br[time_indices, depth_indices, row_indices_roi, col_indices_roi] = ( + batch_vals + ) + nm[time_indices, depth_indices, row_indices_roi, col_indices_roi] = ( + np.isnan(batch_vals) + ) br[nm] = 0 core_t = curr_size - 2 @@ -244,9 +248,9 @@ def finalize_activity_image( act_im: np.ndarray, sel_pix_idxs: np.ndarray, nan_ct: np.ndarray, - ref_r: np.ndarray = None, - ref_c: np.ndarray = None, - ref_d: np.ndarray = None, + ref_r: np.ndarray | None = None, + ref_c: np.ndarray | None = None, + ref_d: np.ndarray | None = None, phase_window_height: int = 7, phase_window_width: int = 11, phase_tolerance: float = 0.0, @@ -278,7 +282,7 @@ def finalize_activity_image( Flattened selected-pixel indices into ``act_im``'s grid. nan_ct : ndarray of shape (n_sel,) Per-selected-pixel rho NaN fraction from - :func:`giant_python.bandsilo.background.mask_high_nan_rho`. + :func:`giant_python.extraction.band.activity.mask_high_nan_rho`. ref_r, ref_c : ndarray, optional Reference-center row and column for each superpixel, expressed in the activity-image coordinate frame. The pipeline shifts these by @@ -305,18 +309,12 @@ def finalize_activity_image( """ nan_mask = np.full_like(act_im, True, dtype=bool) valid_sel_pix = np.flatnonzero(nan_ct <= 0.5) - nan_mask[ - np.unravel_index(sel_pix_idxs[valid_sel_pix], nan_mask.shape) - ] = False + nan_mask[np.unravel_index(sel_pix_idxs[valid_sel_pix], nan_mask.shape)] = ( + False + ) act_im[nan_mask] = np.nan if ref_r is not None and ref_c is not None: - # Legacy trajectory-following filter: uncomment this call and comment - # out the phase-matched call below to switch back. The original helper - # is retained below for comparison and regression tests. - # med_act_im = snake_aligned_nanmedian( - # act_im, np.flatnonzero(~nan_mask), ref_r, ref_c, - # ) med_act_im = phase_matched_nanmedian( act_im, np.flatnonzero(~nan_mask), @@ -338,8 +336,16 @@ def finalize_activity_image( window_height = ( phase_window_height if ref_r is not None and ref_c is not None else 5 ) - vertical_interior = ndimage.minimum_filter1d( - valid_input, size=window_height, axis=1, mode="constant", cval=0, + # With no output override, ndimage preserves the boolean input dtype. + vertical_interior = cast( + NDArray[np.bool_], + ndimage.minimum_filter1d( + valid_input, + size=window_height, + axis=1, + mode="constant", + cval=0, + ), ) vertical_edge = valid_input & ~vertical_interior act_im = act_im - med_act_im @@ -348,12 +354,13 @@ def finalize_activity_image( return act_im -def phase_matched_nanmedian( +# Preserve scientific parity of phase assignment and frozen fallback passes. +def phase_matched_nanmedian( # noqa: C901 image: np.ndarray, valid_pix_idxs: np.ndarray, ref_r: np.ndarray, ref_c: np.ndarray, - ref_d: np.ndarray = None, + ref_d: np.ndarray | None = None, height: int = 7, width: int = 11, phase_tolerance: float = 0.0, @@ -393,11 +400,20 @@ def phase_matched_nanmedian( """ image = np.asarray(image) if image.ndim != 3 or any(size == 0 for size in image.shape): - raise ValueError("image must have non-empty shape (depth, row, column)") + raise ValueError( + "image must have non-empty shape (depth, row, column)" + ) for name, value in (("height", height), ("width", width)): - if not isinstance(value, (int, np.integer)) or value < 1 or value % 2 == 0: + if ( + not isinstance(value, (int, np.integer)) + or value < 1 + or value % 2 == 0 + ): raise ValueError(f"{name} must be a positive odd integer") - for name, value in (("min_samples", min_samples), ("chunk_size", chunk_size)): + for name, value in ( + ("min_samples", min_samples), + ("chunk_size", chunk_size), + ): if not isinstance(value, (int, np.integer)) or value < 1: raise ValueError(f"{name} must be a positive integer") if not np.isfinite(phase_tolerance) or phase_tolerance < 0: @@ -418,17 +434,27 @@ def phase_matched_nanmedian( if ref_d.size != ref_r.size: raise ValueError("ref_d must have the same size as ref_r") for coordinates in (ref_c, ref_d): - if not np.all(np.isfinite(coordinates)) or np.any(coordinates != np.rint(coordinates)): - raise ValueError("reference columns and depths must be finite integers") + if not np.all(np.isfinite(coordinates)) or np.any( + coordinates != np.rint(coordinates) + ): + raise ValueError( + "reference columns and depths must be finite integers" + ) ref_c, ref_d = ref_c.astype(np.intp), ref_d.astype(np.intp) selected = np.asarray(valid_pix_idxs) - if selected.ndim != 1 or (selected.size and not np.issubdtype(selected.dtype, np.integer)): - raise ValueError("valid_pix_idxs must be a one-dimensional integer array") + if selected.ndim != 1 or ( + selected.size and not np.issubdtype(selected.dtype, np.integer) + ): + raise ValueError( + "valid_pix_idxs must be a one-dimensional integer array" + ) if np.any(selected < 0) or np.any(selected >= image.size): raise ValueError("valid_pix_idxs contains an out-of-bounds index") selected = np.unique(selected.astype(np.intp)) - result = np.full(image.shape, np.nan, dtype=np.result_type(image.dtype, np.float32)) + result = np.full( + image.shape, np.nan, dtype=np.result_type(image.dtype, np.float32) + ) if not selected.size: return result @@ -437,12 +463,16 @@ def phase_matched_nanmedian( # Group references by (depth, column); unlike the legacy filter, do not # collapse all rows in a column to one shared trajectory. references = {} - for depth, column, row in zip(ref_d[in_bounds], ref_c[in_bounds], ref_r[in_bounds]): + for depth, column, row in zip( + ref_d[in_bounds], ref_c[in_bounds], ref_r[in_bounds] + ): references.setdefault(int(depth * nc + column), []).append(row) depths, rows, cols = np.unravel_index(selected, image.shape) keys = depths * nc + cols order = np.argsort(keys, kind="stable") - boundaries = np.r_[0, np.flatnonzero(np.diff(keys[order])) + 1, selected.size] + boundaries = np.r_[ + 0, np.flatnonzero(np.diff(keys[order])) + 1, selected.size + ] phase = np.full(selected.size, np.nan) for start, stop in zip(boundaries[:-1], boundaries[1:]): positions = order[start:stop] @@ -455,8 +485,10 @@ def phase_matched_nanmedian( left = np.clip(insertion - 1, 0, centers.size - 1) right = np.clip(insertion, 0, centers.size - 1) nearest = np.where( - np.abs(pixel_rows - centers[left]) <= np.abs(pixel_rows - centers[right]), - left, right, + np.abs(pixel_rows - centers[left]) + <= np.abs(pixel_rows - centers[right]), + left, + right, ) phase[positions] = pixel_rows - centers[nearest] @@ -464,32 +496,48 @@ def phase_matched_nanmedian( # finite unselected image pixels must never enter the sample population. row_offsets, col_offsets = np.meshgrid( np.arange(-(height // 2), height // 2 + 1), - np.arange(-(width // 2), width // 2 + 1), indexing="ij", + np.arange(-(width // 2), width // 2 + 1), + indexing="ij", ) row_offsets, col_offsets = row_offsets.ravel(), col_offsets.ravel() values = image.ravel()[selected] outputs = np.flatnonzero(np.isfinite(phase) & np.isfinite(values)) def windows(output_positions): + """Yield chunked spatial lookups and exact selected-pixel support.""" for start in range(0, output_positions.size, chunk_size): - positions = output_positions[start:start + chunk_size] + positions = output_positions[start : start + chunk_size] sample_r = rows[positions, None] + row_offsets sample_c = cols[positions, None] + col_offsets - inside = (sample_r >= 0) & (sample_r < nr) & (sample_c >= 0) & (sample_c < nc) - sample_flat = (depths[positions, None] * nr + sample_r) * nc + sample_c - lookup = np.minimum(np.searchsorted(selected, sample_flat), selected.size - 1) + inside = ( + (sample_r >= 0) + & (sample_r < nr) + & (sample_c >= 0) + & (sample_c < nc) + ) + sample_flat = ( + depths[positions, None] * nr + sample_r + ) * nc + sample_c + lookup = np.minimum( + np.searchsorted(selected, sample_flat), selected.size - 1 + ) yield positions, lookup, inside & (selected[lookup] == sample_flat) for positions, lookup, support in windows(outputs): accept = ( support & np.isfinite(values[lookup]) - & (np.abs(phase[lookup] - phase[positions, None]) <= phase_tolerance) + & ( + np.abs(phase[lookup] - phase[positions, None]) + <= phase_tolerance + ) ) enough = np.count_nonzero(accept, axis=1) >= min_samples if not np.any(enough): continue - samples = np.where(accept[enough], values[lookup[enough]], np.nan).astype(result.dtype) + samples = np.where( + accept[enough], values[lookup[enough]], np.nan + ).astype(result.dtype) medians = np.nanmedian(samples, axis=1) flat = selected[positions[enough]] result.ravel()[flat] = medians @@ -503,8 +551,14 @@ def windows(output_positions): counts = np.count_nonzero(accept, axis=1) enough = counts > 0 # Equivalent to nanmean, without empty-window warnings. - totals = np.sum(np.where(accept, first_medians[lookup], 0), axis=1, dtype=np.float64) - result.ravel()[selected[positions[enough]]] = totals[enough] / counts[enough] + totals = np.sum( + np.where(accept, first_medians[lookup], 0), + axis=1, + dtype=np.float64, + ) + result.ravel()[selected[positions[enough]]] = ( + totals[enough] / counts[enough] + ) return result @@ -572,7 +626,9 @@ def snake_aligned_nanmedian( continue for index in range(start + 1, stop): delta = center_rows[index] - center_rows[index - 1] - center_rows[index] -= np.rint(delta / snake_spacing) * snake_spacing + center_rows[index] -= ( + np.rint(delta / snake_spacing) * snake_spacing + ) all_columns = np.arange(n_cols) insertions = np.searchsorted(columns, all_columns) @@ -587,7 +643,9 @@ def snake_aligned_nanmedian( trajectory = center_rows[nearest] trajectory_segment = segment_at_reference[nearest] - result = np.full_like(image, np.nan, dtype=np.result_type(image, np.float32)) + result = np.full_like( + image, np.nan, dtype=np.result_type(image, np.float32) + ) col_offsets = np.arange(-(width // 2), width // 2 + 1) row_offsets = np.arange(-(height // 2), height // 2 + 1) @@ -598,8 +656,7 @@ def snake_aligned_nanmedian( valid_cols = (sample_cols >= 0) & (sample_cols < n_cols) clipped_cols = np.clip(sample_cols, 0, n_cols - 1) valid_cols &= ( - trajectory_segment[clipped_cols] - == trajectory_segment[cols, None] + trajectory_segment[clipped_cols] == trajectory_segment[cols, None] ) path_delta = np.rint( trajectory[clipped_cols] - trajectory[cols, None] diff --git a/src/giant_python/bandsilo/traces.py b/src/giant_python/extraction/band/traces.py similarity index 93% rename from src/giant_python/bandsilo/traces.py rename to src/giant_python/extraction/band/traces.py index 7ca9aaf..e18bcd1 100644 --- a/src/giant_python/bandsilo/traces.py +++ b/src/giant_python/extraction/band/traces.py @@ -11,27 +11,29 @@ The pure compute (:func:`compute_high_res_traces`) is factored out of the IO wrapper (:func:`get_high_res_traces`, which reads the SLAP2 file lazily via -:mod:`giant_python.bandsilo.trial_data`) so the numerics are testable without -``slap2_utils``. Per-trial ``.npz`` caching and the ``mp.Pool`` fan-out are -deferred to the Phase-8 pipeline driver. +:mod:`giant_python.extraction.band.trial_data`) so the numerics are testable +without ``slap2_utils``. The workflow owns per-trial scheduling. Results use +``TrialTraceResult``, preserving the historical eight-tuple ordering. """ from __future__ import annotations -from typing import List, Optional, Tuple +from typing import List, Optional, Tuple, cast import numpy as np import torch -from .background import build_motion_h_matrices, build_selected_pixel_mask -from .geometry import ref_pixs_to_drc -from .nmf import solve_phi_motion -from .trial_data import nearest_interp, read_band_trial_data +from ...numerics.interpolation import nearest_interp +from .geometry import build_selected_pixel_mask, ref_pixs_to_drc +from .localization import solve_phi_motion +from .operators import build_motion_h_matrices +from .trial_data import read_band_trial_data +from .types import TrialTraceResult def _empty_trace_result( n_sources: int, num_channels: int, n_soma: int -) -> tuple: +) -> TrialTraceResult: """Return the all-empty result tuple for a skipped trial. Parameters @@ -49,7 +51,7 @@ def _empty_trace_result( Same 8-element shape as :func:`compute_high_res_traces`, with zero frames. """ - return ( + return TrialTraceResult( np.full((0, n_sources), np.nan, dtype=np.float32), np.full((0, n_sources), np.nan, dtype=np.float32), np.full((0,), np.nan), @@ -292,7 +294,8 @@ def _compute_global_f( """ channel_sums = [np.sum(data, axis=0)] if num_channels >= 2: - channel_sums.append(np.sum(data2, axis=0)) + # Two-channel acquisitions supply data2; do not coerce/copy it. + channel_sums.append(np.sum(cast(np.ndarray, data2), axis=0)) global_f = np.stack(channel_sums, axis=-1) global_f[~frames_to_keep, :] = np.nan return global_f @@ -332,7 +335,9 @@ def _compute_f_soma( for i, roi_sps in enumerate(soma_sps): f_soma[:, i, 0] = np.nansum(data[roi_sps, :], axis=0) if num_channels >= 2: - f_soma[:, i, 1] = np.nansum(data2[roi_sps, :], axis=0) + f_soma[:, i, 1] = np.nansum( + cast(np.ndarray, data2)[roi_sps, :], axis=0 + ) f_soma[~frames_to_keep, :, :] = np.nan return f_soma @@ -357,7 +362,7 @@ def compute_high_res_traces( dmd_pixels_per_row: int, num_channels: int, soma_sps: List[np.ndarray], -) -> tuple: +) -> TrialTraceResult: """Compute one trial's high-res source traces from loaded arrays. Parameters @@ -458,7 +463,7 @@ def compute_high_res_traces( data, data2, soma_sps, frames_to_keep, num_channels ) - return ( + return TrialTraceResult( phi.numpy(), f0.numpy(), frames, @@ -486,16 +491,20 @@ def _load_high_res_trial_data( tuple ``(data, data2, a_data, frames)`` with count-normalized activity. """ - result = read_band_trial_data( - trial_ix, - True, - dmd_ix, - samp_freq, - super_pixel_ids, - datadr, - trial_table, - num_channels, - all_channels=True, + # The reader returns None only for skipped trials; keep_trial is True. + result = cast( + dict, + read_band_trial_data( + trial_ix, + True, + dmd_ix, + samp_freq, + super_pixel_ids, + datadr, + trial_table, + num_channels, + all_channels=True, + ), ) data = result["data"] / result["data_count"] data2 = None @@ -525,7 +534,7 @@ def get_high_res_traces( dmd_pixels_per_row: int, num_channels: int, soma_sps: List[np.ndarray], -) -> tuple: +) -> TrialTraceResult: """Read and extract one trial's high-res source traces. Skipped trials (``keep_trial`` False) return the empty result immediately; diff --git a/src/giant_python/bandsilo/trial_data.py b/src/giant_python/extraction/band/trial_data.py similarity index 76% rename from src/giant_python/bandsilo/trial_data.py rename to src/giant_python/extraction/band/trial_data.py index 8e67a93..3fa0c63 100644 --- a/src/giant_python/bandsilo/trial_data.py +++ b/src/giant_python/extraction/band/trial_data.py @@ -1,103 +1,31 @@ """Read low-resolution superpixel activity for band-scan trials. Ported from ``extractSLAP2IntegrationSources.py`` (``get_trial_data`` ref -715-842, ``nearest_interp``/``fast_dilation`` ref 166-205, and the per-trial -low-res caching block ref 1851-1892). The SLAP2 file open (``slap2_utils``) -and the OpenCV dilation fallback import their heavy dependencies lazily so -importing this module stays cheap. The accumulation core, downsample-frame -grid, interpolation, fast dilation, and the low-res assembly/cache IO are all -testable without ``slap2_utils``. +715-842 and the per-trial low-res caching block ref 1851-1892). Vendor file +construction belongs to shared SLAP2 IO; alignment inputs normalize motion +at the band boundary. The original reader lifetime and accumulation order are +unchanged. Shared interpolation and morphology utilities live in +:mod:`giant_python.numerics`. """ from __future__ import annotations import os -import re from typing import Optional import numpy as np -from .hdf5 import load_alignment_data_h5 +from ...io.slap2 import open_slap2_file +from .inputs import load_alignment_data_h5 - -def nearest_interp( - x: np.ndarray, xp: np.ndarray, yp: np.ndarray -) -> np.ndarray: - """Nearest-neighbor interpolation (MATLAB ``interp1(...,'nearest')``). - - Parameters - ---------- - x : ndarray - Query points. - xp : ndarray - Sample positions (assumed sorted ascending). - yp : ndarray - Sample values aligned with ``xp``. - - Returns - ------- - ndarray - ``yp`` sampled at the nearest ``xp`` for each ``x``. - """ - if len(xp) == 1: - return yp - x_bds = xp[:-1] / 2.0 + xp[1:] / 2.0 - idx = np.searchsorted(x_bds, x, side="left") - idx = np.clip(idx, 0, len(xp) - 1) - return yp[idx] - - -def fast_dilation( - mask: np.ndarray, - kernel: Optional[np.ndarray] = None, - iterations: int = 1, -) -> np.ndarray: - """Binary-dilate the trailing two axes of ``mask``. - - The common 3x3 / single-iteration case is handled with a pure-numpy shift - accumulation (fast, no OpenCV); other cases fall back to per-slice - ``cv2.dilate`` over all leading axes. - - Parameters - ---------- - mask : ndarray - Boolean (or 0/1) array; the last two axes are the image plane. - kernel : ndarray, optional - Structuring element; defaults to a 7x7 block. - iterations : int - Number of dilation iterations (generic path only). - - Returns - ------- - ndarray of bool - The dilated mask. - """ - if kernel is None: - kernel = np.ones((7, 7), np.uint8) - - if iterations == 1 and kernel.shape == (3, 3) and np.all(kernel): - m = mask.astype(bool, copy=False) - pad = [(0, 0)] * (m.ndim - 2) + [(1, 1), (1, 1)] - p = np.pad(m, pad, mode="constant", constant_values=False) - out = np.zeros_like(m, dtype=bool) - h, w = m.shape[-2], m.shape[-1] - for dr in range(3): - r_end = dr + h - for dc in range(3): - c_end = dc + w - out |= p[..., dr:r_end, dc:c_end] - return out - - import cv2 - - out = np.empty_like(mask, dtype=bool) - for idx in np.ndindex(mask.shape[:-2]): - out[idx] = cv2.dilate( - mask[idx].astype(np.uint8, copy=False), - kernel, - iterations=iterations, - ).astype(bool, copy=False) - return out +__all__ = [ + "accumulate_superpixel_data", + "assemble_lowres_data", + "compute_ds_frames", + "load_lowres_data", + "read_band_trial_data", + "save_lowres_data", +] def compute_ds_frames( @@ -268,29 +196,6 @@ def accumulate_superpixel_data( return acc -def _open_slap2_file(path: str): # pragma: no cover - requires slap2_utils - """Open a SLAP2 data file (``MultiDataFiles`` for CYCLE globs else single). - - Parameters - ---------- - path : str - Absolute path to the SLAP2 ``.dat`` (or CYCLE-pattern) file. - - Returns - ------- - object - A ``slap2_utils`` data-file object. - """ - import importlib - - import slap2_utils - - importlib.reload(slap2_utils) - if re.search(r"CYCLE\d+", path): - return slap2_utils.MultiDataFiles(path) - return slap2_utils.DataFile(path) - - def read_band_trial_data( trial_ix: int, keep_trial: bool, @@ -345,7 +250,7 @@ def read_band_trial_data( first_line = trial_table["first_line"][dmd_ix, trial_ix] last_line = trial_table["last_line"][dmd_ix, trial_ix] - data_file = _open_slap2_file(os.path.join(datadr, source_fn)) + data_file = open_slap2_file(os.path.join(datadr, source_fn)) dt = 1.0 / samp_freq / data_file.metaData.linePeriod_s ds_frames = compute_ds_frames(first_line, last_line, dt) diff --git a/src/giant_python/extraction/band/types.py b/src/giant_python/extraction/band/types.py new file mode 100644 index 0000000..86a6e26 --- /dev/null +++ b/src/giant_python/extraction/band/types.py @@ -0,0 +1,34 @@ +"""Small runtime records used by band extraction, without copying arrays.""" + +from dataclasses import dataclass +from typing import NamedTuple + +import numpy as np + + +class TrialTraceResult(NamedTuple): + """One trial's named traces in a compact, immutable record. + + Source arrays are (frames, sources); global fluorescence is + (frames, channels) and ROI fluorescence is (frames, ROIs, channels). + Reference offsets and online shifts are (row, column, z) arrays, each + with one value per frame. Only output assembly negates reference offsets + into saved MATLAB displacement. Skipped trials have zero frames. + """ + + d_f: np.ndarray + f0_ls: np.ndarray + frame_line_idxs: np.ndarray + selected_pixels: np.ndarray + global_f: np.ndarray + reference_offsets: tuple[np.ndarray, np.ndarray, np.ndarray] + online_shifts: tuple[np.ndarray, np.ndarray, np.ndarray] + user_roi_f: np.ndarray + + +@dataclass(frozen=True) +class ResolvedAcquisition: + """Acquisition metadata resolved once without mutating science options.""" + + num_channels: int + align_hz: dict[str, float] diff --git a/src/giant_python/extraction/band/workflow.py b/src/giant_python/extraction/band/workflow.py new file mode 100644 index 0000000..873a332 --- /dev/null +++ b/src/giant_python/extraction/band/workflow.py @@ -0,0 +1,591 @@ +"""Canonical band extraction driver, from prepared inputs to eager results. + +Numerical ordering follows the original BandSILo driver. Acquisition metadata, +execution policy and annotations are separate from scientific configuration; +result assembly and persistence belong to their canonical owners. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Optional, Union + +import numpy as np + +from ...io.experiment_summary import write_summary +from ...io.hdf5 import to_serializable +from ...models.experiment_summary import ExperimentSummary, PathSummary +from ...models.params import ( + AnnotationInput, + AnnotationOptions, + BandParamsInput, + BandSiloParams, + ExecutionInput, + ExecutionOptions, + resolve_band_options, +) +from ...models.trial_table import TrialTable +from ...numerics.peaks import get_act_im_peaks +from ...parallel import map_trials +from ...progress import log, progress +from . import activity +from . import background as bg +from . import geometry as geo +from . import inputs, motion_binning, operators +from . import summary_images as si +from . import trial_data as td +from .annotation import ( + build_user_roi_geometry, + resolve_interactivity, + resolve_user_rois, +) +from .inputs import compute_keep_trials, load_trial_table, read_align_info +from .localization import fit_sources +from .result_assembly import ( + PathResult, + _dmd_user_rois, + assemble_path_summary, +) +from .traces import get_high_res_traces +from .types import ResolvedAcquisition + + +def _resolve_session_user_rois( + result_dr: str, + trial_table: dict, + lookup: dict, + annotations: AnnotationOptions, +) -> Optional[dict]: + """Load session annotations or invoke the configured drawing fallback.""" + if not annotations.enabled: + return None + user_roi_geo, ref_files = build_user_roi_geometry(trial_table, lookup) + return resolve_user_rois( + Path(result_dr) / "annotations", + trial_table["n_dmds"], + user_roi_geo, + interactive=resolve_interactivity(annotations.interactive), + ref_files=ref_files, + ) + + +def extract_band_sources( + input: Union[str, Path, TrialTable], + params: BandParamsInput = None, + *, + execution: ExecutionInput = None, + annotations: AnnotationInput = None, +) -> ExperimentSummary: + """Extract all paths from an exact filename or an already loaded table. + + Loaded models are prepared directly, without inferring or reopening a + conventional filename. Channel inference is local; caller scientific and + policy objects remain unchanged. Results are persisted under the table's + savedr/source_extraction directory and returned with every path intact. + """ + params, run, roi = resolve_band_options(params, execution, annotations) + log(f"Loading trial table {input}", run.verbose) + trial_table = load_trial_table(input) + result_dr = str(trial_table["savedr"]) + src_extr_dr = Path(result_dr) / "source_extraction" + src_extr_dr.mkdir(parents=True, exist_ok=True) + output_path = src_extr_dr / "experiment_summary.h5" + + lookup = inputs.load_lookup_table( + Path(result_dr) / "motion_correction" / "bandRegLookupTable.h5", + trial_table["n_dmds"], + ) + psf = inputs.load_psf(params.psf_dilation, trial_table["n_dmds"]) + trial_table["keep_trials"] = compute_keep_trials( + trial_table["fn_adata"], trial_table["filename"], trial_table["datadr"] + ) + align_hz, inferred_channels = read_align_info( + trial_table["fn_adata"], + trial_table["keep_trials"], + trial_table["n_dmds"], + ) + num_channels = ( + params.num_channels + if params.num_channels is not None + else inferred_channels + ) + if num_channels is None: + raise ValueError( + "num_channels could not be determined from the alignment data " + "(no kept trials?); set BandSiloParams.num_channels explicitly." + ) + acquisition = ResolvedAcquisition(num_channels, align_hz) + user_rois = _resolve_session_user_rois(result_dr, trial_table, lookup, roi) + + n_dmds = trial_table["n_dmds"] + log(f"Extracting sources for {n_dmds} DMD path(s)", run.verbose) + paths: list[PathSummary] = [] + for dmd_ix in range(n_dmds): + paths.append( + assemble_path_summary( + _process_dmd( + dmd_ix, + trial_table, + params, + lookup, + psf, + user_rois, + acquisition=acquisition, + execution=run, + annotations=roi, + ), + path_index=dmd_ix, + ) + ) + + params_serializable = to_serializable( + _params_dict(params, acquisition, roi) + ) + log(f"Writing {output_path}", run.verbose) + summary = ExperimentSummary(paths=paths, params=params_serializable) + write_summary(summary, output_path) + return summary + + +def _params_dict( + params: BandSiloParams, + acquisition: ResolvedAcquisition, + annotations: AnnotationOptions, +) -> dict: + """Historical persisted parameter names, with locally resolved metadata.""" + return { + "numChannels": acquisition.num_channels, + "analyzeHz": params.analyze_hz, + "denoiseWindow_s": params.denoise_window_s, + "baselineWindow_s": params.baseline_window_s, + "backgroundInterpolation": params.background_interpolation, + "decayTau_s": params.decay_tau_s, + "dXY": params.d_xy, + "sparse_fac": params.sparse_fac, + "vif": params.vif, + "peakth": params.peakth, + "peak_buffer": params.peak_buffer, + "draw_user_rois": annotations.enabled, + "operator": annotations.operator, + } + + +def _process_dmd( + dmd_ix: int, + trial_table: dict, + params: BandSiloParams, + lookup: dict, + psf: dict, + user_rois: Optional[dict] = None, + *, + acquisition: ResolvedAcquisition, + execution: ExecutionOptions, + annotations: AnnotationOptions, +) -> PathResult: + """Read, localize and extract one path without changing numerical order.""" + key = f"DMD{dmd_ix + 1}" + log(f"Processing {key}", execution.verbose) + soma_masks, soma_labels, soma_sps = _dmd_user_rois(user_rois, key) + datadr = str(trial_table["datadr"]) + n_trials = trial_table["filename"].shape[1] + if execution.max_trials is not None: + n_trials = min(n_trials, execution.max_trials) + log(f"Debug: limiting to first {n_trials} trial(s)", execution.verbose) + num_channels = acquisition.num_channels + align_hz = acquisition.align_hz[key] + keep_trials = trial_table["keep_trials"] + + ref_stack, _, _ = inputs.load_reference_stack( + datadr, trial_table["ref_stack"], dmd_ix + ) + dmd_pixels_per_column = ref_stack.shape[2] + dmd_pixels_per_row = ref_stack.shape[3] + yx_shape = (dmd_pixels_per_column, dmd_pixels_per_row) + fastz_to_refz = lookup["fastZ2RefZ"][key] + num_fast_zs = fastz_to_refz.shape[0] + all_sp_ids = lookup["allSuperPixelIDs"][key] + num_super_pixels = all_sp_ids.shape[0] + n_pixels = num_fast_zs * dmd_pixels_per_column * dmd_pixels_per_row + + smi = geo.build_subsample_matrix_inds( + all_sp_ids, lookup["sparseMaskInds"][key] + ) + psf2d = psf[key] + sparse_h_inds, sparse_h_vals = operators.build_sparse_h( + smi, psf2d, dmd_pixels_per_column, dmd_pixels_per_row + ) + ref_d, ref_c, ref_r = geo.ref_pixs_to_drc( + smi[:, 0], dmd_pixels_per_column, dmd_pixels_per_row + ) + + def _read(trial_ix): + """Read one low-resolution trial in acquisition order.""" + return td.read_band_trial_data( + trial_ix, + bool(keep_trials[dmd_ix, trial_ix]), + dmd_ix, + align_hz, + all_sp_ids, + datadr, + trial_table, + num_channels, + ) + + low = td.assemble_lowres_data( + map_trials( + _read, + range(n_trials), + execution.max_workers, + desc=f"{key} reading trials" if execution.verbose else None, + ), + num_channels, + ) + low_res_data_norm = low["lowResData"] / low["lowResDataCt"] + v_im = 1.0 / low["lowResDataCt"] + low2 = ( + low["lowResData2"] / low["lowResDataCt2"] + if num_channels >= 2 + else None + ) + unique_motion, mot_inds = motion_binning.bin_motion( + low["lowResMotionR"], low["lowResMotionC"], low["lowResMotionZ"] + ) + median_z = np.median(low["lowResMotionZ"]) + _, frames_to_keep = motion_binning.select_motion_bins( + unique_motion, mot_inds, low["lowResMotionZ"] + ) + mean_im = si.compute_mean_image( + low_res_data_norm, + unique_motion, + mot_inds, + frames_to_keep, + ref_d, + ref_r, + ref_c, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + num_channels, + low2, + ) + umyx, mot_inds_yx = motion_binning.bin_motion_yx( + low["lowResMotionR"], low["lowResMotionC"], frames_to_keep + ) + _, sel_pix_idxs = geo.build_selected_pixel_mask( + umyx, + ref_d, + ref_r, + ref_c, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + psf2d, + ) + pixel_coords = geo.pixel_coords_from_idxs( + sel_pix_idxs, dmd_pixels_per_column, dmd_pixels_per_row + ) + background = _estimate_background( + low_res_data_norm, + umyx, + mot_inds_yx, + sel_pix_idxs, + ref_d, + ref_r, + ref_c, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + psf2d, + num_super_pixels, + align_hz, + params, + execution=execution, + ) + data_std, _, _ = bg.fit_noise_variance_model( + low_res_data_norm, background, v_im, params.vif + ) + residual = bg.compute_residual(low_res_data_norm, background, data_std) + act_im, act_im_peaks, a, source_params, source_snr = _localize( + residual, + umyx, + mot_inds_yx, + sel_pix_idxs, + pixel_coords, + n_pixels, + ref_d, + ref_r, + ref_c, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + psf2d, + num_super_pixels, + sparse_h_inds, + sparse_h_vals, + align_hz, + params, + execution=execution, + ) + + def _traces(trial_ix): + """Extract high-res traces using this trial's background slice.""" + return get_high_res_traces( + ( + trial_ix, + bool(keep_trials[dmd_ix, trial_ix]), + background[:, low["lowResTrialID"] == trial_ix], + ), + dmd_ix, + params.analyze_hz, + all_sp_ids, + datadr, + trial_table, + smi, + sparse_h_inds, + sparse_h_vals, + a, + umyx, + np.arange(umyx.shape[0]), + median_z, + psf2d, + num_super_pixels, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + num_channels, + soma_sps, + ) + + trace_results = map_trials( + _traces, + range(n_trials), + execution.max_workers, + desc=f"{key} extracting traces" if execution.verbose else None, + ) + trace_results = [r for r in trace_results if r.d_f.shape[0] > 0] + return PathResult( + a=a, + source_params=source_params, + source_snr=source_snr, + act_im=act_im, + mean_im=mean_im, + act_im_peaks=act_im_peaks, + trace_results=trace_results, + z_depths=fastz_to_refz, + num_fast_zs=num_fast_zs, + dmd_pixels_per_column=dmd_pixels_per_column, + dmd_pixels_per_row=dmd_pixels_per_row, + num_channels=num_channels, + denoise_window=int( + np.ceil(params.denoise_window_s * params.analyze_hz) + ), + baseline_window=int( + np.ceil(params.baseline_window_s * params.analyze_hz) + ), + draw_user_rois=annotations.enabled, + soma_masks=soma_masks, + soma_labels=soma_labels, + yx_shape=yx_shape, + ) + + +def _estimate_background( + low_res_data_norm, + umyx, + mot_inds_yx, + sel_pix_idxs, + ref_d, + ref_r, + ref_c, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + psf2d, + num_super_pixels, + align_hz, + params, + *, + execution=None, +): + """Compose interpolation and rolling baseline on the original grid.""" + execution = ExecutionOptions() if execution is None else execution + log("Estimating background", execution.verbose) + # Preserve the original preparation order, even though these kernels are + # not consumed by the interpolation-based background reconstruction. + operators.expand_psf(psf2d) + interp_data = np.full( + (sel_pix_idxs.shape[0], low_res_data_norm.shape[1]), + np.nan, + dtype=np.float32, + ) + for z in progress( + range(num_fast_zs), + desc="Interpolating planes", + verbose=execution.verbose, + ): + z_idxs, sel_2d = geo.selected_pixels_2d_for_plane( + sel_pix_idxs, z, dmd_pixels_per_column, dmd_pixels_per_row + ) + if z_idxs.size == 0: + continue + ref_z = np.flatnonzero(ref_d == z) + interp_data[z_idxs], _, _ = bg.build_interp_data( + low_res_data_norm[ref_z], + ref_r[ref_z], + ref_c[ref_z], + sel_2d, + umyx, + mot_inds_yx, + method=params.background_interpolation, + ) + baseline_window = bg.baseline_window_frames( + align_hz, params.baseline_window_s + ) + interp_background = bg.compute_rolling_baseline( + interp_data, baseline_window + ) + return bg.assemble_background( + interp_background, + umyx, + mot_inds_yx, + sel_pix_idxs, + ref_d, + ref_r, + ref_c, + low_res_data_norm.shape[0], + low_res_data_norm.shape[1], + dmd_pixels_per_column, + dmd_pixels_per_row, + ) + + +def _localize( + residual, + umyx, + mot_inds_yx, + sel_pix_idxs, + pixel_coords, + n_pixels, + ref_d, + ref_r, + ref_c, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + psf2d, + num_super_pixels, + sparse_h_inds, + sparse_h_vals, + align_hz, + params, + *, + execution=None, +): + """Compose rho, activity and localization; preserve DoG and RNG order.""" + execution = ExecutionOptions() if execution is None else execution + log("Computing activity image and localizing sources", execution.verbose) + h_mots = operators.build_motion_h_matrices( + sparse_h_inds, + sparse_h_vals, + umyx, + sel_pix_idxs, + num_super_pixels, + dmd_pixels_per_row, + ) + # compute_rho normalizes center/surround columns separately. The PSF still + # determines its valid-column dilation/erosion mask, not the DoG widths. + dog_center_sd = 0.9 + dog_surround_sd = 4.5 + center_kernel, center_kernel_ctr = operators.gaussian_kernel_2d( + dog_center_sd + ) + surround_kernel, surround_kernel_ctr = operators.gaussian_kernel_2d( + dog_surround_sd + ) + d_mats, d_mats_exp = [], [] + for z in progress( + range(num_fast_zs), + desc="Computing D matrices", + verbose=execution.verbose, + ): + _, sel_2d = geo.selected_pixels_2d_for_plane( + sel_pix_idxs, z, dmd_pixels_per_column, dmd_pixels_per_row + ) + d_mats.append( + operators.build_convolution_matrix( + sel_2d, center_kernel, center_kernel_ctr + ) + ) + d_mats_exp.append( + operators.build_convolution_matrix( + sel_2d, surround_kernel, surround_kernel_ctr + ) + ) + rho = activity.compute_rho( + residual, + mot_inds_yx, + umyx, + h_mots, + d_mats, + d_mats_exp, + sel_pix_idxs, + ref_d, + ref_r, + ref_c, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + psf2d, + verbose=execution.verbose, + ) + nan_ct = activity.mask_high_nan_rho(rho) + rho = activity.smooth_rho( + rho, + activity.decay_kernel_1d(params.decay_tau_s, align_hz), + verbose=execution.verbose, + ) + act_im = si.accumulate_activity_image( + rho, + sel_pix_idxs, + num_fast_zs, + dmd_pixels_per_column, + dmd_pixels_per_row, + verbose=execution.verbose, + ) + # Only final median geometry follows the dominant retained motion bin. + # Dropped frames (-1) do not vote; rho/NMF retain original scan centers. + motion_counts = np.bincount(mot_inds_yx[mot_inds_yx >= 0]) + dominant_motion = umyx[np.argmax(motion_counts)] + act_im = si.finalize_activity_image( + act_im, + sel_pix_idxs, + nan_ct, + ref_r=ref_r + int(dominant_motion[0]), + ref_c=ref_c + int(dominant_motion[1]), + ref_d=ref_d, + ) + source_seeds = get_act_im_peaks( + act_im, peak_th=params.peakth, buffer_size=params.peak_buffer + ) + act_im_peaks = source_seeds.copy() + print(f"Number of detected sources: {act_im_peaks.shape[0]}") + result = fit_sources( + source_seeds, + residual, + h_mots, + umyx, + mot_inds_yx, + sel_pix_idxs, + pixel_coords, + n_pixels, + params.d_xy, + params.sparse_fac, + verbose=execution.verbose, + ) + return ( + act_im, + act_im_peaks, + result["A"], + result["source_params"], + result["source_snr"], + ) diff --git a/src/giant_python/gui/__init__.py b/src/giant_python/gui/__init__.py index a1538d6..32c7c48 100644 --- a/src/giant_python/gui/__init__.py +++ b/src/giant_python/gui/__init__.py @@ -1,10 +1,4 @@ -"""Interactive GUI for ROI annotation. +"""Parameter and ROI editors with toolkit imports deferred until use. -Import-isolated: the headless pipeline never imports this package, so -``pip install giant-python`` works on a server without GUI dependencies. The -GUI extras (e.g. napari) are pulled in via ``pip install giant-python[gui]``. +Use the public pipeline annotation service for the complete ROI workflow. """ - -from .draw_rois import DrawROIs, annotate_rois - -__all__ = ["annotate_rois", "DrawROIs"] diff --git a/src/giant_python/gui/draw_rois.py b/src/giant_python/gui/draw_rois.py deleted file mode 100644 index c90239a..0000000 --- a/src/giant_python/gui/draw_rois.py +++ /dev/null @@ -1,52 +0,0 @@ -"""ROI annotation GUI (port of drawROIs.m / annotateROIs.m). - -Intended to be backed by napari. Kept in an isolated subpackage so the -headless pipeline never imports GUI dependencies. -""" - -from __future__ import annotations - -from pathlib import Path -from typing import Union - - -class DrawROIs: - """Interactive ROI annotation tool (port of the drawROIs.m class). - - Lets the user draw polygon / circle / ellipse ROIs over the mean and - activity images and saves them to annotations.h5. - - Parameters - ---------- - trial_table : TrialTable - The session trial table (provides image paths and save directory). - """ - - def __init__(self, trial_table: "object") -> None: - """Store the trial table providing image paths and the save dir.""" - self.trial_table = trial_table - - def show(self) -> None: - """Launch the annotation GUI.""" - raise NotImplementedError - - def save_rois(self, path: Union[str, Path]) -> None: - """Save drawn ROIs to annotations.h5. - - Parameters - ---------- - path : str or Path - Destination path for annotations.h5. - """ - raise NotImplementedError - - -def annotate_rois(trial_table: "object") -> None: - """Launch the ROI annotation GUI for a session (port of annotateROIs.m). - - Parameters - ---------- - trial_table : TrialTable - The session trial table. - """ - raise NotImplementedError diff --git a/src/giant_python/gui/parameters_gui.py b/src/giant_python/gui/parameters_gui.py new file mode 100644 index 0000000..c9b8954 --- /dev/null +++ b/src/giant_python/gui/parameters_gui.py @@ -0,0 +1,118 @@ +"""Existing tkinter BandSILo parameter form, with lazy toolkit imports.""" + +from dataclasses import asdict, replace +from typing import Optional + +import numpy as np + +from ..models.params import ( + AnnotationInput, + AnnotationOptions, + BandParamsInput, + BandSiloParams, + ExecutionInput, + ExecutionOptions, + resolve_band_options, +) + +_PARAM_GUI_FIELDS = ( + ("analyze_hz", "Analyze Hz:"), + ("decay_tau_s", "Decay Tau (s):"), + ("baseline_window_s", "Baseline Window (s):"), + ("denoise_window_s", "Denoise Window (s):"), + ("vif", "Variance Inflation Factor (VIF):"), + ("d_xy", "dXY:"), + ("peakth", "Peak Threshold:"), + ("peak_buffer", "Peak Buffer:"), + ("max_workers", "Max Workers:"), + ("operator", "Operator:"), +) + + +def run_parameter_gui( + params: BandParamsInput = None, + *, + execution: ExecutionInput = None, + annotations: AnnotationInput = None, +) -> Optional[ # pragma: no cover - interactive tkinter dialog + tuple[BandSiloParams, ExecutionOptions, AnnotationOptions] +]: + """Return split option objects, or None on cancellation, without mutation. + + Retains the original form and natural-log sparse-factor control. Toolkit + code is imported only when this function is actually invoked. + """ + import tkinter as tk + from tkinter import messagebox, ttk + + base, run, roi = resolve_band_options(params, execution, annotations) + values = {**asdict(base), **asdict(run), **asdict(roi)} + root = tk.Tk() + root.title("SLAP2 Analysis Parameters") + frame = ttk.Frame(root, padding="10") + frame.grid(row=0, column=0) + entries = {} + row = 0 + for attr, label in _PARAM_GUI_FIELDS: + ttk.Label(frame, text=label).grid(row=row, column=0, sticky=tk.W) + var = tk.StringVar(value=str(values[attr])) + ttk.Entry(frame, textvariable=var, width=15).grid(row=row, column=1) + entries[attr] = var + row += 1 + ttk.Label(frame, text="Sparse Factor (log):").grid( + row=row, column=0, sticky=tk.W + ) + sparse_var = tk.StringVar(value=str(float(np.log(base.sparse_fac)))) + ttk.Entry(frame, textvariable=sparse_var, width=15).grid(row=row, column=1) + row += 1 + ttk.Label(frame, text="Draw User ROIs?").grid( + row=row, column=0, sticky=tk.W + ) + draw_rois_var = tk.BooleanVar(value=roi.enabled) + ttk.Checkbutton(frame, variable=draw_rois_var).grid(row=row, column=1) + row += 1 + result: dict[ + str, + Optional[tuple[BandSiloParams, ExecutionOptions, AnnotationOptions]], + ] = {"options": None} + + def on_ok(): + """Validate controls and return a replacement parameter instance.""" + try: + updates = { + "analyze_hz": float(entries["analyze_hz"].get()), + "decay_tau_s": float(entries["decay_tau_s"].get()), + "baseline_window_s": float(entries["baseline_window_s"].get()), + "denoise_window_s": float(entries["denoise_window_s"].get()), + "vif": float(entries["vif"].get()), + "d_xy": int(entries["d_xy"].get()), + "peakth": float(entries["peakth"].get()), + "peak_buffer": int(entries["peak_buffer"].get()), + "sparse_fac": float(np.exp(float(sparse_var.get()))), + } + result["options"] = ( + replace(base, **updates), + replace(run, max_workers=int(entries["max_workers"].get())), + replace( + roi, + operator=entries["operator"].get(), + enabled=bool(draw_rois_var.get()), + ), + ) + root.destroy() + except ValueError as error: + messagebox.showerror("Error", f"Invalid parameter values: {error}") + + def on_cancel(): + """Discard edits and close the dialog.""" + root.destroy() + + button_frame = ttk.Frame(frame) + button_frame.grid(row=row, column=0, columnspan=2, pady=10) + ttk.Button(button_frame, text="OK", command=on_ok).pack(side=tk.LEFT) + ttk.Button(button_frame, text="Cancel", command=on_cancel).pack( + side=tk.LEFT + ) + root.grab_set() + root.mainloop() + return result["options"] diff --git a/src/giant_python/gui/roi_editor.py b/src/giant_python/gui/roi_editor.py new file mode 100644 index 0000000..8af47b2 --- /dev/null +++ b/src/giant_python/gui/roi_editor.py @@ -0,0 +1,198 @@ +"""The existing OpenCV rectangle selector and tkinter label prompt. + +Accepts prepared band reference geometry and returns in-memory records/masks. +There is no file IO here, and tkinter/cv2 are imported only on drawing paths. +""" + +import numpy as np + +from ..extraction.band.rois import user_roi_superpixel_lists_from_masks + + +def _ask_roi_label( + roi_num: int, + path_num: int, + plane_num: int, +) -> str: # pragma: no cover - interactive tkinter dialog + """Prompt for one ROI label, defaulting to ROI{n} on cancel or failure.""" + import tkinter as tk + from tkinter import simpledialog + + text = None + try: + root = tk.Tk() + root.withdraw() + root.attributes("-topmost", True) + text = simpledialog.askstring( + "ROI label", + f"Label for ROI {roi_num} " + f"(Path {path_num}, fast-Z plane {plane_num}):", + parent=root, + ) + root.destroy() + except Exception as error: # noqa: BLE001 - GUI best-effort + print(f"Could not prompt for ROI label ({error}); using default.") + if not text or not text.strip(): + return f"ROI{roi_num}" + return text.strip() + + +def run_user_roi_selection( + user_roi_geo: dict, n_dmds: int +) -> dict: # pragma: no cover - GUI + """Select ROIs on reference images and return per-DMD selections.""" + import cv2 + + out = { + "user_roi_masks": {}, + "user_roi_superpixels": {}, + "user_roi_labels": {}, + "roi_records": {}, + } + for dmd_ix in range(n_dmds): + key = f"DMD{dmd_ix + 1}" + geo = user_roi_geo[key] + roi_masks, roi_records = _select_rois_for_dmd(cv2, dmd_ix, geo) + out["user_roi_masks"][key] = roi_masks + out["user_roi_superpixels"][key] = ( + user_roi_superpixel_lists_from_masks( + roi_masks, geo["sp_fastz"], geo["sp_rows"], geo["sp_cols"] + ) + ) + out["user_roi_labels"][key] = [ + record["label"] for record in roi_records + ] + out["roi_records"][key] = roi_records + return out + + +def _normalize_plane( + plane: np.ndarray, +) -> np.ndarray: # pragma: no cover - display + """Percentile-normalize a reference plane to an 8-bit image.""" + im = np.nan_to_num(plane, nan=0.0) + vmin = np.percentile(im, 1) + vmax = np.percentile(im, 99.5) + if not np.isfinite(vmin): + vmin = float(np.nanmin(im)) if np.any(np.isfinite(im)) else 0.0 + if not np.isfinite(vmax): + vmax = float(np.nanmax(im)) if np.any(np.isfinite(im)) else 1.0 + if vmax <= vmin: + vmax = vmin + 1.0 + im8 = np.clip((im - vmin) / (vmax - vmin), 0, 1) + return (im8 * 255).astype(np.uint8) + + +def _select_rois_for_dmd(cv2, dmd_ix: int, g: dict): # pragma: no cover - GUI + """Run the per-DMD interactive loop, returning masks and records.""" + ref = g["ref"] + num_ref_z = g["num_ref_z"] + yx_shape = g["yx_shape"] + best_ch = g["best_ch"] + z_map = g["z_map"] + num_fast_z = g["num_fast_z"] + sp_mask = g["sp_mask"] + roi_masks = [] + roi_records = [] + roi_union = np.zeros((num_fast_z, *yx_shape), dtype=bool) + window_name = f"Select user ROI(s) DMD{dmd_ix + 1}" + cv2.namedWindow(window_name, cv2.WINDOW_NORMAL) + cv2.resizeWindow(window_name, 800, 500) + has_z_trackbar = num_fast_z > 1 + if has_z_trackbar: + cv2.createTrackbar("z", window_name, 0, num_fast_z - 1, lambda v: None) + curr_fz = 0 + while True: + if has_z_trackbar: + curr_fz = int( + np.clip( + cv2.getTrackbarPos("z", window_name), 0, num_fast_z - 1 + ) + ) + refz = int(np.clip(z_map[curr_fz], 0, max(0, num_ref_z - 1))) + im8 = _normalize_plane(ref[best_ch, refz]) + disp = _compose_display(cv2, im8, sp_mask[curr_fz], roi_union[curr_fz]) + cv2.imshow(window_name, disp) + keycode = cv2.waitKey(50) & 0xFF + if keycode == ord("e"): + _edit_rois( + cv2, + im8, + sp_mask[curr_fz], + curr_fz, + dmd_ix, + num_fast_z, + yx_shape, + roi_masks, + roi_union, + roi_records, + ) + elif keycode == ord("n"): + curr_fz = min(curr_fz + 1, num_fast_z - 1) + if has_z_trackbar: + cv2.setTrackbarPos("z", window_name, curr_fz) + elif keycode == ord("p"): + curr_fz = max(curr_fz - 1, 0) + if has_z_trackbar: + cv2.setTrackbarPos("z", window_name, curr_fz) + elif keycode in (27, ord("q")): + break + cv2.destroyWindow(window_name) + return roi_masks, roi_records + + +def _compose_display( + cv2, im8, sp_plane, roi_plane +): # pragma: no cover - display + """Overlay the superpixel mask in green and ROI union in red.""" + disp = cv2.cvtColor(im8, cv2.COLOR_GRAY2BGR) + if np.any(sp_plane): + sp_color = np.zeros_like(disp) + sp_color[sp_plane] = (0, 255, 0) + disp = cv2.addWeighted(disp, 0.75, sp_color, 0.25, 0) + if roi_plane.any(): + roi_color = np.zeros_like(disp) + roi_color[roi_plane] = (0, 0, 255) + disp = cv2.addWeighted(disp, 0.7, roi_color, 0.3, 0) + return disp + + +def _edit_rois( + cv2, + im8, + sp_plane, + curr_fz, + dmd_ix, + num_fast_z, + yx_shape, + roi_masks, + roi_union, + roi_records, +): # pragma: no cover - GUI + """Append selected rectangles as polygon records and overlapping masks.""" + edit_name = f"Edit ROIs z={curr_fz + 1}" + edit_disp = _compose_display( + cv2, im8, sp_plane, np.zeros(yx_shape, dtype=bool) + ) + rois = cv2.selectROIs( + edit_name, edit_disp, showCrosshair=True, fromCenter=False + ) + cv2.resizeWindow(edit_name, 800, 500) + if rois is not None and len(rois) > 0: + for x, y, w, h in rois: + roi_num = len(roi_masks) + 1 + y_stop, x_stop = y + h, x + w + mask = np.zeros((num_fast_z, *yx_shape), dtype=bool) + mask[curr_fz, y:y_stop, x:x_stop] = True + roi_masks.append(mask) + roi_union[curr_fz, y:y_stop, x:x_stop] = True + label = _ask_roi_label(roi_num, dmd_ix + 1, curr_fz + 1) + position = np.array( + [[y, x], [y, x + w], [y + h, x + w], [y + h, x]], + dtype=np.float64, + ) + roi_records.append( + {"type": "polygon", "label": label, "position": position} + ) + print(f"Added {len(rois)} ROI(s) at fast-Z {curr_fz + 1}") + cv2.destroyWindow(edit_name) diff --git a/src/giant_python/io/__init__.py b/src/giant_python/io/__init__.py index 742a87f..ac64315 100644 --- a/src/giant_python/io/__init__.py +++ b/src/giant_python/io/__init__.py @@ -6,16 +6,52 @@ transposition when porting from MATLAB. """ -from .hdf5 import load_struct_h5, save_struct_h5 -from .slap2 import get_online_motion, read_band_trial_data, ref_pixs_to_drc -from .tiff import scanimagetiff_data_wrapper, scanimagetiff_wrapper +from importlib import import_module +from typing import TYPE_CHECKING + +from .hdf5 import load_struct_h5 +from .slap2 import ( + Slap2Reader, + load_alignment_data_h5, + open_slap2_file, +) +from .tiff import read_tiff + +if TYPE_CHECKING: + from .annotations import read_annotations_h5, save_annotations_h5 + from .experiment_summary import ( + read_summary, + write_summary, + ) + from .trial_table import read_trial_table + +_SCHEMA_EXPORTS = { + "read_trial_table": ".trial_table", + "read_annotations_h5": ".annotations", + "save_annotations_h5": ".annotations", + "read_summary": ".experiment_summary", + "write_summary": ".experiment_summary", +} + + +def __getattr__(name): + """Load model codecs lazily, avoiding schema/model import cycles.""" + if name not in _SCHEMA_EXPORTS: + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + value = getattr(import_module(_SCHEMA_EXPORTS[name], __name__), name) + globals()[name] = value + return value + __all__ = [ - "get_online_motion", + "Slap2Reader", + "load_alignment_data_h5", "load_struct_h5", - "read_band_trial_data", - "ref_pixs_to_drc", - "save_struct_h5", - "scanimagetiff_data_wrapper", - "scanimagetiff_wrapper", + "open_slap2_file", + "read_annotations_h5", + "read_summary", + "read_tiff", + "read_trial_table", + "save_annotations_h5", + "write_summary", ] diff --git a/src/giant_python/io/annotations.py b/src/giant_python/io/annotations.py new file mode 100644 index 0000000..73e61f9 --- /dev/null +++ b/src/giant_python/io/annotations.py @@ -0,0 +1,91 @@ +"""Raw ROI records, masks and provenance IO without band geometry or UI.""" + +from pathlib import Path +from typing import Optional, Union + +import h5py +import numpy as np + +from .hdf5 import load_struct_h5 + + +def _h5_scalar_str(ds: h5py.Dataset) -> str: + """Read a scalar string, including legacy one-element string arrays.""" + value = ds[()] + if isinstance(value, np.ndarray): + value = value.reshape(-1)[0] + if isinstance(value, bytes): + return value.decode("utf-8", "replace") + return str(value) + + +def read_annotations_h5(path: Union[str, Path]) -> dict: + """Read raw Path groups including n_rois, masks, records and provenance. + + No geometry validation or superpixel mapping occurs here. Missing and + unreadable files raise OSError; annotation orchestration decides policy. + The shared decoder honors row_major and leaves coordinate indexes alone. + """ + return load_struct_h5(path) + + +def save_annotations_h5( + dr: str, + roi_records_by_dmd: dict, + user_roi_masks: dict, + n_dmds: int, + ref_files: Optional[dict] = None, +) -> str: + """Write dr/annotations.h5 and return its path, preserving the band schema. + + Rectangles are polygon records with zero-based [y,x] vertices. Masks are + (fast-Z,Y,X), may overlap, and are stored as compressed uint8. The caller + owns directory creation and the decision to replace an existing file. + """ + path = str(Path(dr) / "annotations.h5") + str_dt = h5py.string_dtype(encoding="utf-8") + ref_files = ref_files or {} + with h5py.File(path, "w") as f: + f["row_major"] = 1 + f["coords_zero_indexed"] = 1 + for dmd_ix in range(n_dmds): + key = f"DMD{dmd_ix + 1}" + group = f.create_group(f"Path{dmd_ix + 1}") + ref_file = ref_files.get(key) + group.create_dataset( + "dr", + data=str(Path(ref_file).parent) if ref_file else "", + dtype=str_dt, + ) + group.create_dataset( + "fn", + data=Path(ref_file).name if ref_file else "", + dtype=str_dt, + ) + records = roi_records_by_dmd.get(key, []) + group.create_dataset("n_rois", data=len(records)) + mask_list = user_roi_masks.get(key, []) + for i, record in enumerate(records): + _write_roi_group(group, i, record, mask_list, str_dt) + return path + + +def _write_roi_group(grp, i: int, rec: dict, mask_list: list, str_dt) -> None: + """Write one roi_### group with the established optional-field policy.""" + group = grp.create_group(f"roi_{i:03d}") + roi_type = rec.get("type", "polygon") + group.create_dataset("type", data=roi_type, dtype=str_dt) + group.create_dataset( + "label", data=rec.get("label", f"ROI{i + 1}"), dtype=str_dt + ) + if i < len(mask_list): + group.create_dataset( + "mask", + data=np.asarray(mask_list[i]).astype(np.uint8), + compression="gzip", + shuffle=True, + ) + if roi_type == "polygon" and "position" in rec: + group.create_dataset( + "position", data=np.asarray(rec["position"], dtype=np.float64) + ) diff --git a/src/giant_python/io/experiment_summary.py b/src/giant_python/io/experiment_summary.py new file mode 100644 index 0000000..6523b60 --- /dev/null +++ b/src/giant_python/io/experiment_summary.py @@ -0,0 +1,257 @@ +"""Codec for the experiment-summary schema emitted by band extraction. + +The canonical interface is ``write_summary(summary, path) -> None`` and +``read_summary(path) -> ExperimentSummary``. +There is no motion-sign conversion here. Existing params/row_major are left +untouched; supplied paths are replaced, and unrelated paths are retained. +""" + +from __future__ import annotations + +from collections.abc import Iterable +from pathlib import Path +from typing import Union + +import h5py +import numpy as np + +from ..models.experiment_summary import ( + FRAME_INFO_KEYS, + ExperimentSummary, + FrameInfo, + PathSummary, + Source, + UserRoi, + Visualizations, +) +from .hdf5 import load_struct_h5, write_dict_to_h5group + +_SOURCE_KEYS = ("profiles", "coords", "dF_ls", "F0", "SNR") + + +def _decode_path(assembled: dict, path_index: int) -> PathSummary: + """Decode schema arrays into an eager path, sharing populated arrays. + + This schema-only adapter does not concatenate traces or compute baselines. + Empty-array templates preserve dimensions/dtypes absent from empty lists. + """ + sources = [ + Source( + profile=assembled["profiles"][j], + coords=assembled["coords"][j], + df_ls=assembled["dF_ls"][j], + f0=assembled["F0"][j], + snr=assembled["SNR"][j, 0], + ) + for j in range(assembled["profiles"].shape[0]) + ] + rois = assembled.get("user_rois") + user_rois = [] + if rois is not None: + labels = np.asarray(rois["labels"], dtype=object).reshape(-1) + user_rois = [ + UserRoi( + label=str(label), + mask=rois["mask"][j] if j < rois["mask"].shape[0] else None, + f=rois["F"][j] if j < rois["F"].shape[0] else None, + ) + for j, label in enumerate(labels) + ] + return PathSummary( + path_index=path_index, + sources=sources, + user_rois=user_rois, + visualizations=Visualizations( + mean_im=assembled["mean_im"], + act_im=assembled["act_im"], + act_im_peaks=assembled["act_im_peaks"], + ), + frame_info=FrameInfo( + **{key: assembled[key] for key in FRAME_INFO_KEYS} + ), + global_f=assembled["global_f"], + z_depths=assembled["Z_depths"], + annotation_enabled=rois is not None, + _empty_source_arrays=( + {} if sources else {key: assembled[key] for key in _SOURCE_KEYS} + ), + _empty_roi_arrays=( + {key: rois[key] for key in ("mask", "F")} + if rois is not None and not user_rois + else {} + ), + ) + + +def _stack_required_arrays( + arrays: Iterable[np.ndarray | None], field: str +) -> np.ndarray: + """Stack arrays; identify missing fields by their 0-based index.""" + required: list[np.ndarray] = [] + for index, array in enumerate(arrays): + if array is None: + raise ValueError(f"{field}[{index}] is required; got None") + required.append(array) + return np.stack(required) + + +def _encode_path(path: PathSummary) -> dict: + """Expose band-schema arrays for persistence, without scientific changes. + + Populated lists are stacked so edits to public Source/UserRoi entries are + reflected. Empty results retain their original tensor dimensions/dtypes. + Missing required Source/UserRoi arrays raise ValueError before persistence. + """ + out = { + "Z_depths": path.z_depths, + "global_f": path.global_f, + "mean_im": path.visualizations.mean_im, + "act_im": path.visualizations.act_im, + "act_im_peaks": path.visualizations.act_im_peaks, + **path.frame_info.to_dict(), + } + if path.sources: + out.update( + { + "profiles": _stack_required_arrays( + (s.profile for s in path.sources), "Source.profile" + ), + "coords": _stack_required_arrays( + (s.coords for s in path.sources), "Source.coords" + ), + "dF_ls": _stack_required_arrays( + (s.df_ls for s in path.sources), "Source.df_ls" + ), + "F0": _stack_required_arrays( + (s.f0 for s in path.sources), "Source.f0" + ), + "SNR": np.asarray([s.snr for s in path.sources]).reshape( + -1, 1 + ), + } + ) + elif path._empty_source_arrays: + out.update(path._empty_source_arrays) + else: + raise ValueError( + "Empty source lists require band array shape templates" + ) + if not path.annotation_enabled: + if path.user_rois: + raise ValueError( + "Set annotation_enabled=True to serialize user ROIs" + ) + out["user_rois"] = None + elif path.user_rois: + out["user_rois"] = { + "mask": _stack_required_arrays( + (roi.mask for roi in path.user_rois), "UserRoi.mask" + ), + "labels": [roi.label for roi in path.user_rois], + "F": _stack_required_arrays( + (roi.f for roi in path.user_rois), "UserRoi.f" + ), + } + else: + if path._empty_roi_arrays: + empty = path._empty_roi_arrays + else: + # Enabled-but-empty annotations constructed directly by callers. + mean = np.asarray(path.visualizations.mean_im) + global_f = np.asarray(path.global_f) + if mean.ndim != 4 or global_f.ndim != 2: + raise ValueError( + "Empty ROIs require mean_im and global_f geometry" + ) + empty = { + "mask": np.zeros((0, *mean.shape[1:]), dtype=bool), + "F": np.empty((0, *global_f.shape), dtype=global_f.dtype), + } + out["user_rois"] = {**empty, "labels": []} + return out + + +def _write_path_group(f: h5py.File, path: PathSummary) -> None: + """Replace one Path{index+1} group without array sign conversion.""" + assembled = _encode_path(path) + name = f"Path{path.path_index + 1}" + if name in f: + del f[name] + group = f.create_group(name) + group.create_dataset("Z_depths", data=assembled["Z_depths"]) + sources = group.create_group("sources") + spatial = sources.create_group("spatial") + for key in ("profiles", "coords"): + spatial.create_dataset(key, data=assembled[key]) + temporal = sources.create_group("temporal") + for key in ("dF_ls", "F0", "SNR"): + temporal.create_dataset(key, data=assembled[key]) + frame_group = group.create_group("frame_info") + for key in FRAME_INFO_KEYS: + frame_group.create_dataset(key, data=assembled[key]) + group.create_group("global").create_dataset( + "F", data=assembled["global_f"] + ) + vis = group.create_group("visualizations") + for key in ("act_im", "mean_im", "act_im_peaks"): + vis.create_dataset(key, data=assembled[key]) + if assembled["user_rois"] is not None: + rois = assembled["user_rois"] + roi_group = group.create_group("user_rois") + roi_group.create_dataset("mask", data=rois["mask"]) + roi_group.create_dataset( + "labels", + data=np.asarray(rois["labels"], dtype=object).reshape(-1, 1), + dtype=h5py.string_dtype(encoding="utf-8"), + ) + roi_group.create_dataset("F", data=rois["F"]) + + +def write_summary(summary: ExperimentSummary, path: Union[str, Path]) -> None: + """Persist models, retaining existing params, flags and unrelated paths. + + Path identity comes exclusively from PathSummary.path_index. Required + source/ROI arrays are validated before replacing each existing path group. + """ + indexes = [item.path_index for item in summary.paths] + if len(set(indexes)) != len(indexes): + raise ValueError("Summary contains duplicate path_index values") + with h5py.File(path, "a") as f: + if "row_major" not in f: + f["row_major"] = 1 + if "params" not in f: + write_dict_to_h5group( + f.create_group("params"), + summary.params or {}, + ) + for item in summary.paths: + _write_path_group(f, item) + + +def read_summary(path: Union[str, Path]) -> ExperimentSummary: + """Read emitted band paths eagerly, respecting row_major and saved signs. + + Unrecognized root groups are ignored. Paths are sorted by their numeric + identity, not lexical order. This is not a general standard-SILo codec. + """ + raw = load_struct_h5(path) + names = sorted( + (key for key in raw if key.startswith("Path") and key[4:].isdigit()), + key=lambda key: int(key[4:]), + ) + paths = [] + for name in names: + group = raw[name] + sources = group["sources"] + rois = group.get("user_rois") + assembled = { + "Z_depths": group["Z_depths"], + **sources["spatial"], + **sources["temporal"], + **group["frame_info"], + **group["visualizations"], + "global_f": group["global"]["F"], + "user_rois": rois, + } + paths.append(_decode_path(assembled, int(name[4:]) - 1)) + return ExperimentSummary(paths=paths, params=raw.get("params")) diff --git a/src/giant_python/io/hdf5.py b/src/giant_python/io/hdf5.py index 6b21e43..bdaabc3 100644 --- a/src/giant_python/io/hdf5.py +++ b/src/giant_python/io/hdf5.py @@ -1,11 +1,13 @@ -"""Generic struct <-> HDF5 (de)serialization (port of load/saveStructFromH5). +"""MATLAB-style HDF5 decoding and experiment-parameter group writing. This module is the foundation of MATLAB interoperability: the model ``from_h5`` / ``to_h5`` methods build on these helpers. Round-trip tests against real MATLAB-produced ``.h5`` files are the acceptance criterion. -Serialization conventions (shared with GIAnT-MATLAB; see the package README) ------------------------------------------------------------------------- +Serialization conventions +------------------------- +Shared with GIAnT-MATLAB; see the package README. + * Writers set ``/row_major`` to ``1`` (Python / h5py) or ``0`` (MATLAB). * Dimension tuples in the README match h5py ``shape`` when ``row_major=1``. * Vectors documented as ``1 x N`` or ``N x 1`` are written as rank-2 @@ -15,7 +17,7 @@ """ from pathlib import Path -from typing import Union +from typing import Union, cast import h5py import numpy as np @@ -129,25 +131,73 @@ def load_struct_h5(path: Union[str, Path]) -> dict: with h5py.File(path, "r") as f: row_major = False if "row_major" in f: - row_major = bool(np.asarray(f["row_major"][()]).reshape(-1)[0]) + flag = cast(h5py.Dataset, f["row_major"]) + row_major = bool(np.asarray(flag[()]).reshape(-1)[0]) return _read_group(f, row_major) -def save_struct_h5(struct: dict, path: Union[str, Path]) -> None: - """Write a (possibly nested) dict to an HDF5 file as a MATLAB-style struct. +def to_serializable(val): + """Convert nested NumPy values to ordinary Python parameter values.""" + if isinstance(val, dict): + return {k: to_serializable(v) for k, v in val.items()} + if isinstance(val, (list, tuple)): + return type(val)(to_serializable(v) for v in val) + if isinstance(val, np.ndarray): + return val.tolist() + if isinstance(val, np.generic): + return val.item() + return val - Recursively writes nested dictionaries to HDF5 groups/datasets so that the - result is readable by GIAnT-MATLAB. Corresponds to saveStructToH5.m in - GIAnT-MATLAB. - Sets ``/row_major = 1``. Rank-2 vector shapes (``1 x N`` / ``N x 1``) are - preserved on disk; singleton dimensions are not squeezed. +def _write_preserved_array(grp: h5py.Group, key: str, value) -> None: + """Write a NumPy value with its original shape and numeric dtype.""" + arr = np.asarray(value) + if arr.dtype.kind in ("U", "S", "O"): + grp.create_dataset( + key, + data=np.asarray(_decode_strings(arr), dtype=object), + dtype=h5py.string_dtype(encoding="utf-8"), + ) + else: + grp.create_dataset(key, data=arr) - Parameters - ---------- - struct : dict - Nested dictionary to serialize. - path : str or Path - Destination path for the HDF5 file. + +def _write_sequence(grp: h5py.Group, key: str, value) -> None: + """Skip empty sequences and preserve persisted element stringification.""" + arr = np.asarray(value) + if arr.size == 0: + return + if arr.dtype.kind in ("U", "S", "O"): + grp.create_dataset( + key, + data=np.array([str(x) for x in value], dtype=object), + dtype=h5py.string_dtype(encoding="utf-8"), + ) + else: + grp.create_dataset(key, data=arr) + + +def write_dict_to_h5group(grp: h5py.Group, d: dict) -> None: + """Write params, skipping empty sequences/None and stringifying unknowns. + + This is deliberately NOT a general lossless struct serializer. Its policy + preserves the existing experiment-summary params contract. Decoded arrays + retain their dtype/rank (including uint8 boolean parameter scalars). """ - raise NotImplementedError + str_dt = h5py.string_dtype(encoding="utf-8") + for key, v in d.items(): + key = str(key) + if isinstance(v, dict): + write_dict_to_h5group(grp.create_group(key), v) + elif isinstance(v, (np.ndarray, np.generic)): + _write_preserved_array(grp, key, v) + elif isinstance(v, bool): + grp.create_dataset(key, data=np.uint8(v)) + elif isinstance(v, str): + grp.create_dataset(key, data=v, dtype=str_dt) + elif isinstance(v, (int, float)): + grp.create_dataset(key, data=v) + elif isinstance(v, (list, tuple)): + _write_sequence(grp, key, v) + elif v is not None: + grp.create_dataset(key, data=str(v), dtype=str_dt) diff --git a/src/giant_python/io/slap2.py b/src/giant_python/io/slap2.py index b69e810..61d8223 100644 --- a/src/giant_python/io/slap2.py +++ b/src/giant_python/io/slap2.py @@ -1,99 +1,77 @@ -"""SLAP2 binary data I/O (``.dat`` / ``.meta``) and online-motion retrieval. +"""SLAP2 binary reader access and saved alignment-data decoding. -Intended home for the SLAP2 data-file reader (equivalent of MATLAB's -``slap2.Slap2DataFile``) plus the online motion-offset retrieval ported from -getOnlineMotion.m. Also hosts the band-scan superpixel readers -used by the band source-extraction backend. +Band geometry and superpixel reduction belong to the extraction.band backend, +not this raw acquisition adapter. """ -from typing import Optional, Tuple +from pathlib import Path +from typing import Any, Protocol, Union import numpy as np +from .hdf5 import load_struct_h5 -def ref_pixs_to_drc( - ref_pixs: np.ndarray, - dmd_pixels_per_column: int, - dmd_pixels_per_row: int, -) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: - """Map flat reference pixel indices to DMD (depth, column, row) indices. - Port of ``ref_pixs_to_drc`` from ``extractSLAP2IntegrationSources.py``. +def _reshape_1d(src: dict, key: str): + """Flatten an optional raw alignment vector without altering its sign.""" + value = src.get(key) + return None if value is None else np.asarray(value).reshape(-1) - Parameters - ---------- - ref_pixs : ndarray of int - Flat reference-pixel indices. - dmd_pixels_per_column, dmd_pixels_per_row : int - DMD geometry. - Returns - ------- - ref_d, ref_c, ref_r : ndarray of int - Depth, column, and row indices. +def load_alignment_data_h5(path: Union[str, Path]) -> dict: + """Read alignment vectors with the original saved displacement sign. + + Flatten DS/offline/online vectors and expose numChannels/alignHz scalars. + No coordinate normalization or motion negation belongs in this reader. + Band numerical consumers must use extraction.band.inputs instead. """ - raise NotImplementedError - - -def read_band_trial_data( - data_file: object, - ds_frames: np.ndarray, - super_pixel_ids: np.ndarray, - activity_channel: Optional[int] = None, -) -> Tuple[np.ndarray, np.ndarray]: - """Read superpixel-binned activity for one band-scan trial. - - Accumulates weighted line data into per-superpixel time series over the - downsampled frame grid. Port of ``get_trial_data`` from - ``extractSLAP2IntegrationSources.py``. - - Parameters - ---------- - data_file : object - SLAP2 data-file object exposing line/superpixel access. - ds_frames : ndarray - Downsampled frame (line-index) grid. - super_pixel_ids : ndarray - Superpixel id lookup for the DMD being read. - activity_channel : int, optional - Channel to read; ``None`` reads all channels. - - Returns - ------- - data : ndarray of shape (n_superpixels, n_ds_frames) - Weighted-mean superpixel activity. - data_count : ndarray of shape (n_superpixels, n_ds_frames) - Accumulated weights (for normalization). + data = load_struct_h5(path) + online = data.get("slap2", {}) or {} + return { + "DSframes": _reshape_1d(data, "DSframes"), + "motionDSr": _reshape_1d(data, "motionDSr"), + "motionDSc": _reshape_1d(data, "motionDSc"), + "motionDSz": _reshape_1d(data, "motionDSz"), + "onlineYshift": _reshape_1d(online, "onlineMotionYshift"), + "onlineXshift": _reshape_1d(online, "onlineMotionXshift"), + "onlineZshift": _reshape_1d(online, "onlineMotionZshift"), + "numChannels": int(np.asarray(data["numChannels"]).reshape(-1)[0]), + "alignHz": float(np.asarray(data["alignHz"]).reshape(-1)[0]), + } + + +class Slap2Reader(Protocol): + """Minimal raw-reader surface consumed by the existing band reducer. + + Line/cycle indexes passed to getLineData remain 1-based. The owning + caller retains the reader for the entire batched reduction; this protocol + deliberately promises neither a close method nor context-manager support. """ - raise NotImplementedError - - -def get_online_motion( - data_file: object, - ds_frames: np.ndarray, -) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: - """Retrieve online motion correction offsets from a SLAP2 data file. - - Reads the per-frame X, Y, and Z motion correction shifts that were - computed online during SLAP2 acquisition by inspecting per-line headers - in the data file. Corresponds to getOnlineMotion.m in GIAnT-MATLAB. - - Parameters - ---------- - data_file : object - SLAP2 data file object exposing line-header access (e.g. a - ``Slap2DataFile`` instance). - ds_frames : ndarray of int - Line indices of the downsampled frames for which to retrieve - offsets. - - Returns - ------- - online_x_shift : ndarray of shape (n_frames,) - Online motion correction X (column) shifts, in pixels. - online_y_shift : ndarray of shape (n_frames,) - Online motion correction Y (row) shifts, in pixels. - online_z_shift : ndarray of shape (n_frames,) - Online motion correction Z (axial) shifts, in micrometres. + + header: dict + metaData: Any + numCycles: int + lineDataNumElements: Any + lineSuperPixelIDs: Any + lineFastZIdxs: Any + + def getLineData(self, line_indices, cycle_indices, channels=None): + """Read a batch using the vendor reader's unchanged contract.""" + + +def open_slap2_file(path: Union[str, Path]) -> Slap2Reader: + """Lazily open DataFile/MultiDataFiles; return ownership to the caller. + + Retains the original CYCLE detection and module reload. No reads, close, + copying, or early resource disposal are performed here. """ - raise NotImplementedError + import importlib + import re + + import slap2_utils + + importlib.reload(slap2_utils) + path = str(path) + if re.search(r"CYCLE\d+", path): + return slap2_utils.MultiDataFiles(path) + return slap2_utils.DataFile(path) diff --git a/src/giant_python/io/tiff.py b/src/giant_python/io/tiff.py index 30c3177..43625f6 100644 --- a/src/giant_python/io/tiff.py +++ b/src/giant_python/io/tiff.py @@ -1,61 +1,17 @@ -"""ScanImage TIFF reading (port of ScanImageTiffWrapper / ...DataWrapper). - -Also the intended home for the fast TIFF writers (ports of Fast_Tiff_Write / -Fast_BigTiff_Write) used to emit registered movies. -""" +"""Native TIFF array reading without acquisition-specific transformations.""" from pathlib import Path -from typing import List, Tuple, Union +from typing import Union import numpy as np -def scanimagetiff_wrapper( - fn: Union[str, Path], -) -> Tuple[np.ndarray, List[str]]: - """Read image data and per-frame metadata from a ScanImage TIFF file. - - Opens the file with a ScanImage TIFF reader, extracts the image stack - via :func:`scanimagetiff_data_wrapper`, and returns the per-frame - metadata strings. Corresponds to ScanImageTiffWrapper.m in GIAnT-MATLAB. +def read_tiff(path: Union[str, Path]) -> np.ndarray: + """Read a TIFF eagerly, preserving native dtype, rank and page ordering. - Parameters - ---------- - fn : str or Path - Path to the ScanImage TIFF file. - - Returns - ------- - data : ndarray - Image stack read from the file. - meta : str - File-level metadata string embedded in the TIFF. + Reference intensity scaling, channel reshaping and PSF cropping are + caller policies, not generic TIFF decoding. Import tifffile only on use. """ - raise NotImplementedError - + import tifffile -def scanimagetiff_data_wrapper( - reader: object, - tif_file: Union[str, Path], -) -> np.ndarray: - """Read a ScanImage TIFF stack, falling back to standard TIFF reading. - - Attempts to read image data via *reader*. On failure, reads all pages of - the TIFF using standard image I/O. Corresponds to - ScanImageTiffDataWrapper.m in GIAnT-MATLAB. - - Parameters - ---------- - reader : object - ScanImage TIFF reader object (e.g. from the ``scanimage-tiff-reader`` - Python package). - tif_file : str or Path - Path to the ScanImage TIFF file used to construct *reader*. - - Returns - ------- - ndarray - Image stack in the native on-disk datatype, shaped - ``(W, H, n_frames)`` to match ScanImage convention. - """ - raise NotImplementedError + return tifffile.imread(path) diff --git a/src/giant_python/io/trial_table.py b/src/giant_python/io/trial_table.py new file mode 100644 index 0000000..bc2b3f3 --- /dev/null +++ b/src/giant_python/io/trial_table.py @@ -0,0 +1,48 @@ +"""Faithful trial-table schema decoding, without runtime file discovery.""" + +from pathlib import Path +from typing import Optional, Union + +import numpy as np + +from .hdf5 import load_struct_h5 + + +def slap2_info_from_raw(slap2: Optional[dict]): + """Decode recognized SLAP2 metadata; keep absent metadata absent.""" + from ..models.trial_table import Slap2Info + + if not slap2: + return None + names = ( + "ref_stack", + "first_line", + "last_line", + "trial_start_time_inferred", + "trial_end_time_from_pc", + ) + return Slap2Info(**{key: slap2[key] for key in names if key in slap2}) + + +def read_trial_table(path: Union[str, Path]): + """Return an eager TrialTable with source provenance, no path resolution. + + ``source_path`` records the input filename and is not part of disk data. + Numeric ranks are retained; the filename string grid is always rank two. + """ + from ..models.trial_table import TrialTable + + raw = load_struct_h5(path) + return TrialTable( + datadr=Path(raw["datadr"]) if raw.get("datadr") else None, + savedr=Path(raw["savedr"]) if raw.get("savedr") else None, + filename=np.atleast_2d( + np.asarray(raw.get("filename", []), dtype=object) + ), + true_trial_ix=raw.get("true_trial_ix", []), + epoch=raw.get("epoch", []), + slap2_info=slap2_info_from_raw(raw.get("slap2_info")), + motion_correction=raw.get("motion_correction"), + source_extraction=raw.get("source_extraction"), + source_path=Path(path).absolute(), + ) diff --git a/src/giant_python/math/__init__.py b/src/giant_python/math/__init__.py deleted file mode 100644 index 987d791..0000000 --- a/src/giant_python/math/__init__.py +++ /dev/null @@ -1,28 +0,0 @@ -"""Numerical kernels for GIAnT-Python. - -Pure, side-effect-free functions ported closely from GIAnT-MATLAB so they can -be validated against MATLAB outputs function-by-function. Watch MATLAB<->NumPy -axis-order and 0- vs 1-based indexing differences when porting. -""" - -from .baseline import compute_f0 -from .interpolation import interp_frame -from .peaks import detect_peaks_2d, gaussian_peaks_integrated, get_act_im_peaks -from .registration import ( - dft_register_clipped, - xcorr2_nans, - xcorr2_nans3d, - xcorr2_nans_weighted, -) - -__all__ = [ - "compute_f0", - "detect_peaks_2d", - "dft_register_clipped", - "gaussian_peaks_integrated", - "get_act_im_peaks", - "interp_frame", - "xcorr2_nans", - "xcorr2_nans3d", - "xcorr2_nans_weighted", -] diff --git a/src/giant_python/math/baseline.py b/src/giant_python/math/baseline.py deleted file mode 100644 index 7550644..0000000 --- a/src/giant_python/math/baseline.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Fluorescence baseline (F0) estimation. - -Shared by both source-extraction backends. Ports ``computeF0.m`` (GIAnT-MATLAB) -/ ``compute_f0`` from ``extractSLAP2IntegrationSources.py``. -""" - -import numpy as np - - -def compute_f0( - f_in: np.ndarray, - denoise_window: int, - hull_window: int, -) -> np.ndarray: - """Estimate the slowly varying baseline F0 of fluorescence traces. - - Median-filters to reduce noise, applies a rolling convex-hull-like min - envelope on decimated grids, discards doubtful samples near NaNs, then - smooths/fills and PCHIP-interpolates back to the full time grid. - - Parameters - ---------- - f_in : ndarray of shape (T, ...) - Fluorescence with time along axis 0; NaNs allowed. - denoise_window : int - Median-filter window, in time samples. - hull_window : int - Window controlling the convex-hull-like min envelope. - - Returns - ------- - ndarray - Baseline F0, same shape as *f_in*. - """ - raise NotImplementedError diff --git a/src/giant_python/math/deconv.py b/src/giant_python/math/deconv.py deleted file mode 100644 index 0f2205b..0000000 --- a/src/giant_python/math/deconv.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Temporal deconvolution of fluorescence traces into events (SILo).""" - -import numpy as np - - -def deconvolve_trace( - trace: np.ndarray, - tau_s: float, - frame_rate_hz: float, -) -> np.ndarray: - """Deconvolve a fluorescence trace into an inferred event train. - - Removes the indicator decay kinetics (single-exponential with time - constant *tau_s*) to recover an event/spike train. Corresponds to the - deconvolution step in GIAnT-MATLAB source extraction. - - Parameters - ---------- - trace : ndarray of shape (n_frames,) - Denoised dF/F trace. - tau_s : float - Indicator decay time constant, in seconds. - frame_rate_hz : float - Acquisition/analysis frame rate, in Hz. - - Returns - ------- - ndarray of shape (n_frames,) - Inferred non-negative event train. - """ - raise NotImplementedError diff --git a/src/giant_python/math/interpolation.py b/src/giant_python/math/interpolation.py deleted file mode 100644 index de6ef52..0000000 --- a/src/giant_python/math/interpolation.py +++ /dev/null @@ -1,40 +0,0 @@ -"""Frame interpolation with photon-count weighting (port of interpFrame.m).""" - -from typing import Tuple - -import numpy as np - - -def interp_frame( - m1: np.ndarray, - view_c: np.ndarray, - view_r: np.ndarray, - freshness: np.ndarray, -) -> Tuple[np.ndarray, np.ndarray]: - """Interpolate an image and compute the per-pixel variance multiplier. - - Performs bilinear interpolation of a 2-D image at arbitrary coordinates, - weighting each contributing pixel by its *freshness* (effective photon - count). Also returns a variance multiplier proportional to the local - measurement noise. Corresponds to interpFrame.m in GIAnT-MATLAB. - - Parameters - ---------- - m1 : ndarray of shape (H, W) - Source image to interpolate. - view_c : ndarray - Column coordinates at which to evaluate the interpolation. - view_r : ndarray - Row coordinates at which to evaluate the interpolation. - freshness : ndarray of shape (H, W) - Effective sample count for each pixel of *m1*. - - Returns - ------- - im : ndarray - Interpolated image evaluated at (*view_r*, *view_c*). - v_im : ndarray - Variance multiplier; multiply a pixel intensity by this value to - obtain a quantity proportional to the pixel's variance. - """ - raise NotImplementedError diff --git a/src/giant_python/math/nmf.py b/src/giant_python/math/nmf.py deleted file mode 100644 index 1feded3..0000000 --- a/src/giant_python/math/nmf.py +++ /dev/null @@ -1,41 +0,0 @@ -"""Non-negative matrix factorization for source decomposition (SILo).""" - -from typing import Tuple - -import numpy as np - - -def nmf_decompose( - activity: np.ndarray, - n_sources: int, - lambda_: float = 0.1, - phi: float = 0.0, - n_iter: int = 100, -) -> Tuple[np.ndarray, np.ndarray]: - """Decompose pixel activity into spatial profiles and temporal traces. - - Sparse, spatially regularized non-negative matrix factorization used by - the SILo source-extraction stage. Corresponds to the NMF refinement step - in GIAnT-MATLAB source extraction. - - Parameters - ---------- - activity : ndarray of shape (n_pixels, n_frames) - Per-pixel activity matrix. - n_sources : int - Number of sources to extract. - lambda_ : float - Sparsity weight. - phi : float - Spatial-coherence weight. - n_iter : int - Number of update iterations. - - Returns - ------- - spatial : ndarray of shape (n_pixels, n_sources) - Spatial profiles. - temporal : ndarray of shape (n_sources, n_frames) - Temporal traces. - """ - raise NotImplementedError diff --git a/src/giant_python/math/peaks.py b/src/giant_python/math/peaks.py deleted file mode 100644 index 34333da..0000000 --- a/src/giant_python/math/peaks.py +++ /dev/null @@ -1,95 +0,0 @@ -"""Activity-image peak detection via integrated-Gaussian fitting. - -Shared by both source-extraction backends (standard and band). Ports -``getActImPeaks.m`` (GIAnT-MATLAB) / ``get_act_im_peaks`` and its -integrated-Gaussian helpers from ``extractSLAP2IntegrationSources.py``. -""" - -from typing import Optional - -import numpy as np - - -def gaussian_peaks_integrated( - theta: np.ndarray, - yx: np.ndarray, -) -> np.ndarray: - """Evaluate integrated isotropic 2-D Gaussians over unit pixels. - - Port of ``gaussianPeaksIntegrated`` from getActImPeaks.m. - - Parameters - ---------- - theta : ndarray of shape (N, 4) - Per-peak parameters ``[amp, mu_y, mu_x, sigma]``. - yx : ndarray of shape (M, 2) - Pixel-centre coordinates ``[y, x]``. - - Returns - ------- - ndarray of shape (M,) - Predicted values at the given pixel centres. - """ - raise NotImplementedError - - -def detect_peaks_2d( - act_im_2d: np.ndarray, - exclusion_mask: np.ndarray, - mu_bg: float, - sigma_bg: float, - peak_thresh: float, - peak_th: float, -) -> np.ndarray: - """Detect Gaussian peaks in a single activity-image plane. - - Performs initial local-maximum detection followed by iterative - residual peak finding with bounded Levenberg-Marquardt curve fitting. - - Parameters - ---------- - act_im_2d : ndarray of shape (H, W) - One plane of the activity image; may contain NaNs. - exclusion_mask : ndarray of shape (H, W) - Pixels to exclude from detection. - mu_bg, sigma_bg : float - Background mean and (MAD-normalised) standard deviation. - peak_thresh : float - Absolute amplitude threshold (``mu_bg + peak_th * sigma_bg``). - peak_th : float - Threshold in MAD-normalised standard deviations. - - Returns - ------- - ndarray of shape (N, 4) - Fitted peaks ``[amp, mu_y, mu_x, sigma]`` (``(0, 4)`` if none). - """ - raise NotImplementedError - - -def get_act_im_peaks( - act_im: np.ndarray, - peak_th: float = 3.0, - exclusion_mask: Optional[np.ndarray] = None, -) -> np.ndarray: - """Find Gaussian peaks across a 3-D ``(Z, H, W)`` activity image. - - Background statistics and the detection threshold are computed once - across all planes for uniform sensitivity, then each plane is processed - independently via :func:`detect_peaks_2d`. - - Parameters - ---------- - act_im : ndarray of shape (Z, H, W) - Activity image; may contain NaNs. - peak_th : float - Threshold in MAD-normalised standard deviations. - exclusion_mask : ndarray, optional - Mask broadcastable to ``(Z, H, W)`` of pixels to exclude. - - Returns - ------- - ndarray of shape (N, 3) - Source seeds ``[z, mu_y, mu_x]`` (``(0, 3)`` if none detected). - """ - raise NotImplementedError diff --git a/src/giant_python/math/registration.py b/src/giant_python/math/registration.py deleted file mode 100644 index 3cf2eab..0000000 --- a/src/giant_python/math/registration.py +++ /dev/null @@ -1,124 +0,0 @@ -"""FFT-based and NaN-aware cross-correlation registration kernels. - -Ports of dftregistration_clipped.m, xcorr2_nans.m, xcorr2_nans3d.m, and -xcorr2_nans_weighted.m from GIAnT-MATLAB. -""" - -from typing import Tuple - -import numpy as np - - -def xcorr2_nans_weighted( - frame: np.ndarray, - freshness: np.ndarray, - template: np.ndarray, - shifts_center: np.ndarray, - d_shift: float, -) -> Tuple[np.ndarray, float]: - """Perform weighted, NaN-aware local normalised cross-correlation. - - Estimates the sub-pixel translation between *frame* and *template* by - evaluating the weighted normalised cross-correlation over a local search - window of +/-*d_shift* pixels centred on *shifts_center*. Corresponds to - xcorr2_nans_weighted.m in GIAnT-MATLAB. - - Parameters - ---------- - frame : ndarray of shape (H, W) - Floating image to align; may contain NaNs. - freshness : ndarray of shape (H, W) - Per-pixel effective sample count (photon-count weighting) for - *frame*. - template : ndarray of shape (H, W) - Fixed reference image. - shifts_center : ndarray of shape (2,) - ``[row, col]`` centre of the local search window. - d_shift : float - Half-width of the search window in pixels (rounded to integer). - - Returns - ------- - motion : ndarray of shape (2,) - Estimated ``[row, col]`` shift of *frame* relative to *template*. - r : float - Peak normalised cross-correlation coefficient. - """ - raise NotImplementedError - - -def xcorr2_nans( - frame: np.ndarray, - template: np.ndarray, -) -> np.ndarray: - """Compute NaN-aware 2-D normalised cross-correlation. - - Port of xcorr2_nans.m in GIAnT-MATLAB. - - Parameters - ---------- - frame : ndarray of shape (H, W) - Image to align; may contain NaNs. - template : ndarray of shape (H, W) - Fixed reference image. - - Returns - ------- - ndarray - Cross-correlation surface. - """ - raise NotImplementedError - - -def xcorr2_nans3d( - frame: np.ndarray, - template: np.ndarray, -) -> np.ndarray: - """Compute NaN-aware 3-D normalised cross-correlation. - - Port of xcorr2_nans3d.m in GIAnT-MATLAB. - - Parameters - ---------- - frame : ndarray of shape (Z, H, W) - Volume to align; may contain NaNs. - template : ndarray of shape (Z, H, W) - Fixed reference volume. - - Returns - ------- - ndarray - Cross-correlation volume. - """ - raise NotImplementedError - - -def dft_register_clipped( - buf_fft: np.ndarray, - target_fft: np.ndarray, - usfac: int = 1, - max_shift: float = np.inf, -) -> np.ndarray: - """Estimate sub-pixel translation by upsampled DFT cross-correlation. - - Port of dftregistration_clipped.m in GIAnT-MATLAB: efficient sub-pixel - image registration via upsampled DFT, with the shift clipped to - ``+/-max_shift``. - - Parameters - ---------- - buf_fft : ndarray - FFT of the image to register. - target_fft : ndarray - FFT of the reference image. - usfac : int - Upsampling factor (sub-pixel resolution = ``1 / usfac``). - max_shift : float - Maximum allowed shift magnitude, in pixels. - - Returns - ------- - ndarray - Estimated ``[row, col]`` shift. - """ - raise NotImplementedError diff --git a/src/giant_python/math/variance.py b/src/giant_python/math/variance.py deleted file mode 100644 index fa15e11..0000000 --- a/src/giant_python/math/variance.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Poisson variance modeling and activity-image construction (SILo).""" - -import numpy as np - - -def activity_image( - movie: np.ndarray, - photon_scale: float, -) -> np.ndarray: - """Build a heteroscedastic-noise-normalized activity image from a movie. - - Uses a Poisson-based variance model (variance proportional to mean photon - count, via *photon_scale*) to weight temporal fluctuations, yielding an - image that highlights active sources. Corresponds to the activity-image - construction in GIAnT-MATLAB source extraction. - - Parameters - ---------- - movie : ndarray of shape (n_frames, H, W) - Motion-corrected movie. - photon_scale : float - Photons per digital unit, used to scale the variance model. - - Returns - ------- - ndarray of shape (H, W) - Activity image. - """ - raise NotImplementedError diff --git a/src/giant_python/models/__init__.py b/src/giant_python/models/__init__.py index d2e802d..90ce112 100644 --- a/src/giant_python/models/__init__.py +++ b/src/giant_python/models/__init__.py @@ -1,22 +1,33 @@ """Typed data models for GIAnT-Python. -These dataclasses are the single source of truth for the on-disk HDF5 schema -shared with GIAnT-MATLAB. Each model knows how to (de)serialize itself via -``from_h5`` / ``to_h5`` so that files written by either toolbox remain -interchangeable. +Scientific configuration is distinct from run and annotation policy. +Implemented model codecs delegate to shared IO. """ -from .alignment import AlignmentData -from .experiment import ExperimentSummary, Source, UserRoi, Visualizations -from .params import AlignParams, SiloParams, set_params +from .experiment_summary import ( + ExperimentSummary, + FrameInfo, + PathSummary, + Source, + UserRoi, + Visualizations, +) +from .params import ( + AnnotationOptions, + BandSiloParams, + ExecutionOptions, + resolve_band_options, +) from .trial_table import Slap2Info, TrialTable __all__ = [ - "AlignmentData", - "AlignParams", + "AnnotationOptions", + "BandSiloParams", + "ExecutionOptions", "ExperimentSummary", - "set_params", - "SiloParams", + "FrameInfo", + "PathSummary", + "resolve_band_options", "Slap2Info", "Source", "TrialTable", diff --git a/src/giant_python/models/alignment.py b/src/giant_python/models/alignment.py deleted file mode 100644 index 4499eb9..0000000 --- a/src/giant_python/models/alignment.py +++ /dev/null @@ -1,107 +0,0 @@ -"""Alignment data model (mirrors the ``aData`` struct / *_ALIGNMENTDATA.h5).""" - -from __future__ import annotations - -from dataclasses import dataclass, field -from pathlib import Path -from typing import Optional, Union - -import numpy as np - - -@dataclass -class AlignmentData: - """Per-trial motion-correction output written to ``*_ALIGNMENTDATA.h5``. - - Mirrors the ``aData`` struct in GIAnT-MATLAB. Field presence depends on - which registration backend wrote the file (StripRegistration, - MultiRoiRegistration, or BandRegistration); see the GIAnT-python README - Pipeline Outputs section. - - Attributes - ---------- - num_channels : int or None - Number of channels in the recording (``numChannels``). - frametime : float or None - Seconds per downsampled frame. - align_hz : float or None - Frame rate (Hz) at which alignment was performed (``alignHz``). - motion_ds_c, motion_ds_r : ndarray or None - Per-downsampled-frame column / row shifts (``motionDSc`` / - ``motionDSr``). - motion_ds_z : ndarray or None - Per-downsampled-frame Z shift (``motionDSz``); always written by - BandRegistration; by MultiRoiRegistration only when - ``refStackTemplate`` is enabled; never by StripRegistration. - mean_im : ndarray or None - Per-channel mean registered image, shaped ``(channels, rows, cols)`` - (``meanIM``); not written by BandRegistration. - rec_neg_err : ndarray or None - Reconstruction error per frame (``recNegErr``); not written by - BandRegistration. - motion_c, motion_r : ndarray or None - Column / row shifts upsampled to raw frame rate (``motionC`` / - ``motionR``); StripRegistration / Bergamo only. - motion_z : ndarray or None - Reserved (``motionZ``); not written by any current backend. - brightness_ds : ndarray or None - Per-channel brightness/scaling at the selected motion shift, shaped - ``(nDSframes, channels)`` (``brightnessDS``); BandRegistration only. - log_likelihood_ds : ndarray or None - Peak log-likelihood of the motion match per downsampled frame - (``logLikelihoodDS``); BandRegistration only. - ds_frames : ndarray or None - 1-indexed line indices of the downsampled frames (``DSframes``); - SLAP2 only (MultiRoiRegistration and BandRegistration). - registration_failed : bool - Whether registration failed for this trial (``registrationFailed``); - SLAP2 only. - slap2 : dict or None - SLAP2-only sub-struct. Always may include - ``onlineMotion{X,Y,Z}shift``; MultiRoiRegistration also writes - ``varFacDS``, ``Z_depths``, ``cropRow``/``cropCol``, - ``viewC``/``viewR``, ``trimRows``/``trimCols``. - """ - - num_channels: Optional[int] = None - frametime: Optional[float] = None - align_hz: Optional[float] = None - motion_ds_c: Optional[np.ndarray] = None - motion_ds_r: Optional[np.ndarray] = None - motion_ds_z: Optional[np.ndarray] = None - mean_im: Optional[np.ndarray] = None - rec_neg_err: Optional[np.ndarray] = None - motion_c: Optional[np.ndarray] = None - motion_r: Optional[np.ndarray] = None - motion_z: Optional[np.ndarray] = None - brightness_ds: Optional[np.ndarray] = None - log_likelihood_ds: Optional[np.ndarray] = None - ds_frames: Optional[np.ndarray] = None - registration_failed: bool = False - slap2: Optional[dict] = field(default=None) - - @classmethod - def from_h5(cls, path: Union[str, Path]) -> "AlignmentData": - """Load alignment data from a ``*_ALIGNMENTDATA.h5`` file. - - Parameters - ---------- - path : str or Path - Path to the alignment data file. - - Returns - ------- - AlignmentData - The deserialized alignment data. - """ - raise NotImplementedError - - def to_h5(self, path: Union[str, Path]) -> None: - """Write alignment data to a ``*_ALIGNMENTDATA.h5`` file. - - Parameters - ---------- - path : str or Path - Destination path for the alignment data file. - """ - raise NotImplementedError diff --git a/src/giant_python/models/experiment.py b/src/giant_python/models/experiment_summary.py similarity index 56% rename from src/giant_python/models/experiment.py rename to src/giant_python/models/experiment_summary.py index 067474d..8eb01ef 100644 --- a/src/giant_python/models/experiment.py +++ b/src/giant_python/models/experiment_summary.py @@ -2,7 +2,7 @@ from __future__ import annotations -from dataclasses import dataclass, field +from dataclasses import dataclass, field, fields from pathlib import Path from typing import Optional, Union @@ -36,7 +36,7 @@ class Source: df_denoised: Optional[np.ndarray] = None events: Optional[np.ndarray] = None f0: Optional[np.ndarray] = None - snr: Optional[float] = None + snr: Optional[Union[float, np.floating]] = None @dataclass @@ -109,70 +109,86 @@ class Visualizations: @dataclass -class ExperimentSummary: - """Source-extraction output (``exptSummary`` / experiment_summary.h5). +class FrameInfo: + """Eager, typed frame bookkeeping. - The headline result object returned by the extraction stage. Note that - heavy arrays are intended to be disk-backed / lazily loaded; the - :class:`~giant_python.pipeline.pipeline.Pipeline` facade holds a reference - to this object but does not pin its pixel data in memory. + Offline shifts carry saved MATLAB displacement, NOT private band + reference offsets. No conversion occurs in this model or its codec. + Frame line indexes remain 1-based; emitted vectors retain rank two. + ``to_dict`` returns a shallow mapping, sharing the original arrays. + """ - Attributes - ---------- - sources : list of list of Source - Extracted sources, indexed ``[path][source]`` (one inner list per - imaging path / ``Path{n}`` group). - user_rois : list of list of UserRoi - Manually drawn ROIs and their traces, indexed ``[path][roi]``. - visualizations : Visualizations - Summary images (single-path convenience view; see also - per-path structure in ``experiment_summary.h5``). - z_depths : ndarray or None - Z depths per imaging plane (``Path{n}/Z_depths``; SLAP2 only), - shaped ``(fastz,)``. - global_f : ndarray or None - Whole-field fluorescence per channel (``Path{n}/global/F``), shaped - ``(channels, total frames)``. - frame_info : dict or None - Trial/frame bookkeeping for the stitched time series - (``Path{n}/frame_info``): ``offline{X,Y,Z}shifts``, - ``online{X,Y,Z}shifts``, ``trial_num_frames``, ``frame_line_idxs`` - (1-indexed), and ``discard_frames``. - params : dict or None - The SILo parameters used to produce this summary. ``activityChannel`` - is 1-indexed into the recording's channels. + trial_num_frames: Optional[np.ndarray] = None + discard_frames: Optional[np.ndarray] = None + frame_line_idxs: Optional[np.ndarray] = None + offlineXshifts: Optional[np.ndarray] = None + offlineYshifts: Optional[np.ndarray] = None + offlineZshifts: Optional[np.ndarray] = None + onlineXshifts: Optional[np.ndarray] = None + onlineYshifts: Optional[np.ndarray] = None + onlineZshifts: Optional[np.ndarray] = None + + def to_dict(self) -> dict: + """Return all frame fields without copying or changing their arrays.""" + return {item.name: getattr(self, item.name) for item in fields(self)} + + +FRAME_INFO_KEYS = tuple(item.name for item in fields(FrameInfo)) + + +@dataclass +class PathSummary: + """All eager output for one imaging path, identified by a 0-based index. + + Arrays remain NumPy-backed in memory; Source/UserRoi entries may be views + of assembled arrays. ``annotation_enabled`` distinguishes an absent ROI + group from an enabled-but-empty group. Private empty-array templates + retain dimensions and dtypes that cannot be inferred from empty lists; + they never duplicate populated source or ROI tensors. """ - sources: list = field(default_factory=list) - user_rois: list = field(default_factory=list) + path_index: int = 0 + sources: list[Source] = field(default_factory=list) + user_rois: list[UserRoi] = field(default_factory=list) visualizations: Visualizations = field(default_factory=Visualizations) - z_depths: Optional[np.ndarray] = None + frame_info: FrameInfo = field(default_factory=FrameInfo) global_f: Optional[np.ndarray] = None - frame_info: Optional[dict] = None + z_depths: Optional[np.ndarray] = None + annotation_enabled: bool = False + _empty_source_arrays: dict = field(default_factory=dict, repr=False) + _empty_roi_arrays: dict = field(default_factory=dict, repr=False) + + def __post_init__(self): + """Validate path identity without copying any numerical arrays.""" + if ( + not isinstance(self.path_index, (int, np.integer)) + or self.path_index < 0 + ): + raise ValueError( + "path_index must be a nonnegative, 0-based integer" + ) + + +@dataclass +class ExperimentSummary: + """Eager, in-memory extraction output with explicit per-path ownership. + + This is not a lazy or disk-backed model. All imaging data belongs to + entries in ``paths``; only extraction parameters are shared. + """ + + paths: list[PathSummary] = field(default_factory=list) params: Optional[dict] = None @classmethod def from_h5(cls, path: Union[str, Path]) -> "ExperimentSummary": - """Load an experiment summary from experiment_summary.h5. + """Read the currently emitted band-summary schema eagerly.""" + from ..io.experiment_summary import read_summary - Parameters - ---------- - path : str or Path - Path to the experiment_summary.h5 file. - - Returns - ------- - ExperimentSummary - The deserialized experiment summary. - """ - raise NotImplementedError + return read_summary(path) def to_h5(self, path: Union[str, Path]) -> None: - """Write this experiment summary to experiment_summary.h5. - - Parameters - ---------- - path : str or Path - Destination path for the experiment_summary.h5 file. - """ - raise NotImplementedError + """Write band-schema output using the shared IO codec; return None.""" + from ..io.experiment_summary import write_summary + + write_summary(self, path) diff --git a/src/giant_python/models/params.py b/src/giant_python/models/params.py index 99f07db..1665514 100644 --- a/src/giant_python/models/params.py +++ b/src/giant_python/models/params.py @@ -1,158 +1,72 @@ -"""Typed parameter models and the legacy ``set_params`` helper. +"""Scientific configuration and independent execution and annotation policy. -The dataclasses here replace the untyped parameter dicts used in -GIAnT-MATLAB with validated, autocompleting parameter objects. The -:func:`set_params` function is retained as a dict-based compatibility shim -(a direct port of setParams.m) for code paths that still pass raw dicts. +``BandSiloParams`` contains only band science and channel overrides. +``ExecutionOptions`` controls scheduling/debug limits; ``AnnotationOptions`` +controls ROI use and GUI policy. Resolution copies inputs without retuning +the existing algorithm. """ from __future__ import annotations import math -from dataclasses import dataclass -from typing import Optional +from dataclasses import asdict, dataclass, fields +from numbers import Integral, Real +from typing import Optional, Union -@dataclass -class AlignParams: - """Motion-correction parameters (mirrors the registration params struct). - - Attributes - ---------- - align_hz : float - Downsampled alignment rate, in Hz. - maxshift : float - Maximum allowed shift, in pixels. - clip_shift : float or None - Clip threshold for per-frame shift changes. - alpha : float or None - Regularization weight. - n_workers : int - Number of parallel workers for per-trial processing. - overwrite_existing : bool - Recompute trials whose outputs already exist. - is_revolt : bool - Bergamo ReVolt acquisition mode. - """ - - align_hz: float = 80.0 - maxshift: float = 20.0 - clip_shift: Optional[float] = None - alpha: Optional[float] = None - n_workers: int = 1 - overwrite_existing: bool = False - is_revolt: bool = False +def _positive(name: str, value: object, *, integer: bool = False) -> None: + """Reject nonfinite, nonpositive, boolean, or nonnumeric values.""" + kind = Integral if integer else Real + if ( + isinstance(value, bool) + or not isinstance(value, kind) + or not math.isfinite(value) + or value <= 0 + ): + label = "positive integer" if integer else "positive finite number" + raise ValueError(f"{name} must be a {label}") @dataclass -class SiloParams: - """Source-extraction (SILo) parameters (mirrors the SILo params struct). - - Defaults policy - --------------- - Every value parameter carries a concrete default, so a bare - :class:`SiloParams` never lets a ``None`` reach the numerics. ``None`` is - reserved as an explicit "resolve at runtime" sentinel for the few fields - that cannot have a fixed default: ``num_channels`` (read from the - acquisition metadata), ``interactive`` (auto-detected from the - environment), and the standard-SILo-only ``phi`` / ``tau_s`` / - ``photon_scale`` (consumed only by the not-yet-implemented pixel-movie - backend). Each of those is resolved in exactly one place. +class BandSiloParams: + """Scientific settings for the implemented SLAP2 band algorithm. Attributes ---------- - microscope : str - ``"slap2"`` or ``"bergamo"``. - scan_mode : str - SLAP2 scan mode selecting the extraction backend: - ``"standard"`` (pixel-movie SILo) or ``"band"`` (the - band-scan / superpixel variant, paired with BandRegistration). - sigma_px : float - Spatial smoothing sigma, in pixels. - lambda_ : float - NMF sparsity weight (``lambda`` in MATLAB; renamed to avoid the - Python keyword). - phi : float or None - NMF spatial-coherence weight. - tau_s : float or None - Deconvolution time constant, in seconds. - photon_scale : float or None - Photons per digital unit (variance modeling). - peakth : float - Activity-image peak detection threshold (default ``8``). - activity_channel : int - Channel index used for activity detection. - draw_user_rois : bool - Whether to prompt for / use user ROIs; also gates writing the - ``user_rois`` group and per-ROI traces to ``experiment_summary.h5``. - interactive : bool or None - Tri-state override for whether a GUI may be launched when - ``draw_user_rois`` is set but no ``annotations.h5`` exists. ``True`` - forces the drawing GUI, ``False`` forces headless (fail fast with - guidance), and ``None`` (the default) auto-detects (respecting the - ``GIANT_HEADLESS`` env var and stdin being a TTY). See - :func:`giant_python.bandsilo.annotate.resolve_interactivity`. analyze_hz : float - Temporal analysis rate, in Hz (default ``100``; the value the - reference's parameter GUI defaulted to). - - Notes - ----- - The remaining attributes below are specific to the ``"band"`` - (BandSILo) backend and mirror the GUI defaults in - ``extractSLAP2IntegrationSources.py``. They are ignored by the standard - (pixel-movie) SILo backend. - + Temporal analysis rate in Hz. decay_tau_s : float - Calcium-decay time constant, in seconds. - baseline_window_s : float - Rolling-baseline window for dF/F, in seconds. - background_interpolation : str - Interpolation used to map sparse superpixels onto the background image: - ``"cubic"`` (default; natural cubic spline) or ``"linear"``. - denoise_window_s : float - Temporal denoising window, in seconds. + Calcium-decay time constant in seconds. + baseline_window_s, denoise_window_s : float + Baseline and temporal denoising windows in seconds. + background_interpolation : {'cubic', 'linear'} + Sparse-superpixel background interpolation method. vif : float - Variance inflation factor for the noise model. + Noise-model variance inflation factor. d_xy : int - Spatial downsampling / superpixel spacing, in pixels. + Spatial downsampling / superpixel spacing in pixels. sparse_fac : float - NMF sparsity factor (the GUI exposes its natural-log; the default is - ``exp(-3)``). + NMF sparsity factor, not its logarithm. + peakth : float + Peak threshold. Seven preserves the current implementation; this + configuration split does not retune the algorithm. peak_buffer : int - Peak exclusion buffer diameter, in pixels. - max_workers : int - Number of worker processes for per-trial extraction. - num_channels : int or None - Number of acquisition channels; when ``None``, read from the aData - alignment file at runtime. + Peak exclusion buffer diameter in pixels. psf_dilation : int - Dilation size selecting the bundled ``dil-NN.tif`` PSF template. - operator : str - Operator name recorded in the output metadata. - verbose : bool - When set, the band pipeline prints per-stage status messages and shows - tqdm progress bars over its long-running loops (per-trial reads, rho, - activity image, NMF). Off by default so headless / batch runs stay - quiet. See :mod:`giant_python.bandsilo.progress`. - max_trials : int or None - Debug knob: when set, process only the first ``max_trials`` trials per - DMD (both the low-res read and high-res trace extraction). ``None`` - (the default) processes all trials. Use a small value (e.g. ``5``) to - iterate quickly while debugging. + Positive dilation size selecting the bundled PSF template. Template + availability is checked by the backend when loading the PSF. + num_channels : int or None + Positive channel-count override; otherwise infer from acquisition + metadata without modifying this object. + activity_channel : int + Zero-based activity channel. Only channel zero is implemented. + + Notes + ----- + Scheduling, trial limits, logging and annotation policy are deliberately + separate from science. Numerical defaults preserve the band algorithm. """ - microscope: str = "slap2" - scan_mode: str = "standard" - sigma_px: float = 1.5 - lambda_: float = 0.1 - phi: Optional[float] = None - tau_s: Optional[float] = None - photon_scale: Optional[float] = None - peakth: float = 7.0 - activity_channel: int = 0 - draw_user_rois: bool = False - interactive: Optional[bool] = None analyze_hz: float = 100.0 decay_tau_s: float = 0.15 baseline_window_s: float = 4.0 @@ -161,44 +75,145 @@ class SiloParams: vif: float = 1.38 d_xy: int = 5 sparse_fac: float = math.exp(-3.0) + peakth: float = 7.0 peak_buffer: int = 3 - max_workers: int = 6 - num_channels: Optional[int] = None psf_dilation: int = 17 - operator: str = "SLAP2 User" + num_channels: Optional[int] = None + activity_channel: int = 0 + + def __post_init__(self) -> None: + """Validate values supported by the existing numerical path.""" + for name in ( + "analyze_hz", + "decay_tau_s", + "baseline_window_s", + "denoise_window_s", + "vif", + "sparse_fac", + "peakth", + ): + _positive(name, getattr(self, name)) + for name in ("d_xy", "peak_buffer", "psf_dilation"): + _positive(name, getattr(self, name), integer=True) + if self.num_channels is not None: + _positive("num_channels", self.num_channels, integer=True) + if self.background_interpolation not in ("cubic", "linear"): + raise ValueError( + "background_interpolation must be 'cubic' or 'linear'" + ) + if ( + isinstance(self.activity_channel, bool) + or not isinstance(self.activity_channel, Integral) + or self.activity_channel != 0 + ): + raise ValueError("Only activity_channel=0 is implemented") + + +@dataclass +class ExecutionOptions: + """Run policy, separate from scientific configuration. + + Attributes + ---------- + max_workers : int + Positive number of worker processes; preserves the default of six. + verbose : bool + Enable stage messages and progress bars. + max_trials : int or None + Positive debug limit per DMD. ``None`` processes all trials. + """ + + max_workers: int = 6 verbose: bool = False max_trials: Optional[int] = None + def __post_init__(self) -> None: + """Validate scheduling and debug limits.""" + _positive("max_workers", self.max_workers, integer=True) + if self.max_trials is not None: + _positive("max_trials", self.max_trials, integer=True) + if not isinstance(self.verbose, bool): + raise ValueError("verbose must be a bool") + + +@dataclass +class AnnotationOptions: + """ROI inclusion and GUI policy, separate from scientific settings. + + Attributes + ---------- + enabled : bool + Include user ROIs during extraction. + interactive : bool or None + Allow or forbid drawing; ``None`` retains backend auto-detection. + operator : str + Operator name recorded in metadata. + """ + + enabled: bool = False + interactive: Optional[bool] = None + operator: str = "SLAP2 User" + + def __post_init__(self) -> None: + """Validate the explicit and automatic annotation policies.""" + if not isinstance(self.enabled, bool): + raise ValueError("enabled must be a bool") + if self.interactive is not None and not isinstance( + self.interactive, bool + ): + raise ValueError("interactive must be a bool or None") + if not isinstance(self.operator, str): + raise ValueError("operator must be a string") -def set_params( - fn_name: str, - params_in: Optional[dict] = None, -) -> dict: - """Return a parameter dictionary for the named GIAnT function. - Populates default parameters for the specified function and merges any - caller-supplied overrides. Corresponds to setParams.m in GIAnT-MATLAB. - Prefer the typed :class:`AlignParams` / :class:`SiloParams` models for new - code; this shim exists for dict-based compatibility. +BandParamsInput = Optional[Union[BandSiloParams, dict]] +ExecutionInput = Optional[Union[ExecutionOptions, dict]] +AnnotationInput = Optional[Union[AnnotationOptions, dict]] + + +def _option_values( + option: object, + model: type[Union[BandSiloParams, ExecutionOptions, AnnotationOptions]], +) -> dict: + """Copy a typed option or validate a strictly scoped dictionary.""" + if option is None: + return {} + if isinstance(option, model): + return asdict(option) + if isinstance(option, dict): + allowed = {item.name for item in fields(model)} + unknown = option.keys() - allowed + if unknown: + raise TypeError(f"Unknown {model.__name__} fields: {unknown}") + return dict(option) + raise TypeError(f"Expected {model.__name__}, dict, or None") + + +def resolve_band_options( + params: BandParamsInput = None, + execution: ExecutionInput = None, + annotations: AnnotationInput = None, +) -> tuple[BandSiloParams, ExecutionOptions, AnnotationOptions]: + """Resolve independent band science, execution and annotation options. Parameters ---------- - fn_name : str - Name of the GIAnT function whose parameters should be configured. - Supported values: ``"SILo"``, ``"MultiRoiRegistration"``, - ``"StripRegistration"``, ``"BandRegistration"``. - params_in : dict, optional - Caller-supplied parameter overrides. Any key present in *params_in* - takes precedence over the corresponding default value. + params : BandSiloParams, dict or None + Scientific settings only. Routing, execution, annotation and unknown + fields raise ``TypeError`` rather than being dispatched elsewhere. + execution : ExecutionOptions, dict or None + Run options or a dictionary containing only execution fields. + annotations : AnnotationOptions, dict or None + Annotation policy or a dictionary containing only annotation fields. Returns ------- - dict - Parameter dictionary with defaults merged with *params_in*. - - Raises - ------ - ValueError - If *fn_name* is not a recognised GIAnT function name. + tuple of BandSiloParams, ExecutionOptions, AnnotationOptions + Fresh validated objects. Caller inputs are never mutated, including + when channel count or GUI interactivity is later resolved. """ - raise NotImplementedError + return ( + BandSiloParams(**_option_values(params, BandSiloParams)), + ExecutionOptions(**_option_values(execution, ExecutionOptions)), + AnnotationOptions(**_option_values(annotations, AnnotationOptions)), + ) diff --git a/src/giant_python/models/trial_table.py b/src/giant_python/models/trial_table.py index c458d36..ea96bae 100644 --- a/src/giant_python/models/trial_table.py +++ b/src/giant_python/models/trial_table.py @@ -6,15 +6,7 @@ from pathlib import Path from typing import Optional, Union -from ..io import load_struct_h5 - -_SLAP2_FIELDS = ( - "ref_stack", - "first_line", - "last_line", - "trial_start_time_inferred", - "trial_end_time_from_pc", -) +import numpy as np @dataclass @@ -29,17 +21,21 @@ class Slap2Info: ref_stack : dict Reference stack(s), keyed by DMD path, each with image, channels, Zs, and the DMD-pixel-to-sample transform. - first_line, last_line : list of int + first_line, last_line : ndarray or list First/last scanned line index for each trial. - trial_start_time_inferred, trial_end_time_from_pc : list of float + trial_start_time_inferred, trial_end_time_from_pc : ndarray or list Inferred and PC-reported trial timing. """ ref_stack: dict = field(default_factory=dict) - first_line: list = field(default_factory=list) - last_line: list = field(default_factory=list) - trial_start_time_inferred: list = field(default_factory=list) - trial_end_time_from_pc: list = field(default_factory=list) + first_line: Union[np.ndarray, list] = field(default_factory=list) + last_line: Union[np.ndarray, list] = field(default_factory=list) + trial_start_time_inferred: Union[np.ndarray, list] = field( + default_factory=list + ) + trial_end_time_from_pc: Union[np.ndarray, list] = field( + default_factory=list + ) @dataclass @@ -58,29 +54,33 @@ class TrialTable: Directory containing the raw data files. savedr : Path or None Directory in which results are written. - filename : list + filename : ndarray or list File names, shaped ``(n_dmds, n_trials)``. - true_trial_ix : list + true_trial_ix : ndarray or list Trial indices, shaped ``(n_dmds, n_trials)``. - epoch : list + epoch : ndarray or list Epoch numbers, shaped ``(n_dmds, n_trials)``. slap2_info : Slap2Info or None SLAP2-only metadata (``None`` for Bergamo). motion_correction : dict or None Populated by the registration stage (registered filenames, alignment - data filenames, per-trial failure flags, and the ``AlignParams`` used). + data filenames, per-trial failure flags, and registration parameters). source_extraction : dict or None Populated by the extraction stage (analysis params, raw input files). """ datadr: Optional[Path] = None savedr: Optional[Path] = None - filename: list = field(default_factory=list) - true_trial_ix: list = field(default_factory=list) - epoch: list = field(default_factory=list) + filename: Union[np.ndarray, list] = field(default_factory=list) + true_trial_ix: Union[np.ndarray, list] = field(default_factory=list) + epoch: Union[np.ndarray, list] = field(default_factory=list) slap2_info: Optional[Slap2Info] = None motion_correction: Optional[dict] = None source_extraction: Optional[dict] = None + # Provenance only; never a trial-table HDF5 field. + source_path: Optional[Path] = field( + default=None, repr=False, compare=False + ) @classmethod def from_h5(cls, path: Union[str, Path]) -> "TrialTable": @@ -104,45 +104,6 @@ def from_h5(cls, path: Union[str, Path]) -> "TrialTable": TrialTable The deserialized trial table. """ - raw = load_struct_h5(path) - slap2 = raw.get("slap2_info") - return cls( - datadr=Path(raw["datadr"]) if raw.get("datadr") else None, - savedr=Path(raw["savedr"]) if raw.get("savedr") else None, - filename=raw.get("filename"), - true_trial_ix=raw.get("true_trial_ix"), - epoch=raw.get("epoch"), - slap2_info=cls._slap2_info_from_raw(slap2), - motion_correction=raw.get("motion_correction"), - source_extraction=raw.get("source_extraction"), - ) - - @staticmethod - def _slap2_info_from_raw(slap2: Optional[dict]) -> Optional["Slap2Info"]: - """Build a :class:`Slap2Info` from the raw ``slap2_info`` dict. - - Parameters - ---------- - slap2 : dict or None - The faithfully-loaded ``slap2_info`` group (or ``None`` for a - non-SLAP2 experiment). + from ..io.trial_table import read_trial_table - Returns - ------- - Slap2Info or None - The populated metadata (only recognized fields), or ``None``. - """ - if not slap2: - return None - kwargs = {k: slap2[k] for k in _SLAP2_FIELDS if k in slap2} - return Slap2Info(**kwargs) - - def to_h5(self, path: Union[str, Path]) -> None: - """Write this trial table to a MATLAB-compatible trial_table.h5 file. - - Parameters - ---------- - path : str or Path - Destination path for the trial_table.h5 file. - """ - raise NotImplementedError + return read_trial_table(path) diff --git a/src/giant_python/numerics/__init__.py b/src/giant_python/numerics/__init__.py new file mode 100644 index 0000000..9755622 --- /dev/null +++ b/src/giant_python/numerics/__init__.py @@ -0,0 +1,4 @@ +"""Implemented baseline, peak, interpolation and morphology kernels. + +Import individual modules as needed. +""" diff --git a/src/giant_python/bandsilo/baseline.py b/src/giant_python/numerics/baseline.py similarity index 99% rename from src/giant_python/bandsilo/baseline.py rename to src/giant_python/numerics/baseline.py index 3176f2a..d63648d 100644 --- a/src/giant_python/bandsilo/baseline.py +++ b/src/giant_python/numerics/baseline.py @@ -1,4 +1,4 @@ -"""Baseline (F0) estimation and dF/F assembly. +"""Shared baseline (F0) estimation and dF/F assembly. Ported from ``extractSLAP2IntegrationSources.py`` (``_movmean_nan`` / ``computeF0`` ref 217-305, and the dF/F assembly ref 2972-2978). diff --git a/src/giant_python/numerics/interpolation.py b/src/giant_python/numerics/interpolation.py new file mode 100644 index 0000000..5933975 --- /dev/null +++ b/src/giant_python/numerics/interpolation.py @@ -0,0 +1,30 @@ +"""Shared nearest-neighbor interpolation.""" + +import numpy as np + + +def nearest_interp( + x: np.ndarray, xp: np.ndarray, yp: np.ndarray +) -> np.ndarray: + """Nearest-neighbor interpolation (MATLAB ``interp1(...,'nearest')``). + + Parameters + ---------- + x : ndarray + Query points. + xp : ndarray + Sample positions (assumed sorted ascending). + yp : ndarray + Sample values aligned with ``xp``. + + Returns + ------- + ndarray + ``yp`` sampled at the nearest ``xp`` for each ``x``. + """ + if len(xp) == 1: + return yp + x_bds = xp[:-1] / 2.0 + xp[1:] / 2.0 + idx = np.searchsorted(x_bds, x, side="left") + idx = np.clip(idx, 0, len(xp) - 1) + return yp[idx] diff --git a/src/giant_python/numerics/morphology.py b/src/giant_python/numerics/morphology.py new file mode 100644 index 0000000..4e4abbb --- /dev/null +++ b/src/giant_python/numerics/morphology.py @@ -0,0 +1,60 @@ +"""Shared binary morphology on trailing image axes.""" + +from __future__ import annotations + +from typing import Optional + +import numpy as np + + +def fast_dilation( + mask: np.ndarray, + kernel: Optional[np.ndarray] = None, + iterations: int = 1, +) -> np.ndarray: + """Binary-dilate the trailing two axes of ``mask``. + + The common 3x3 / single-iteration case is handled with a pure-numpy shift + accumulation (fast, no OpenCV); other cases fall back to per-slice + ``cv2.dilate`` over all leading axes. + + Parameters + ---------- + mask : ndarray + Boolean (or 0/1) array; the last two axes are the image plane. + kernel : ndarray, optional + Structuring element; defaults to a 7x7 block. + iterations : int + Number of dilation iterations (generic path only). + + Returns + ------- + ndarray of bool + The dilated mask. + """ + if kernel is None: + kernel = np.ones((7, 7), np.uint8) + + if iterations == 1 and kernel.shape == (3, 3) and np.all(kernel): + m = mask.astype(bool, copy=False) + pad = [(0, 0)] * (m.ndim - 2) + [(1, 1), (1, 1)] + p = np.pad(m, pad, mode="constant", constant_values=False) + out = np.zeros_like(m, dtype=bool) + h, w = m.shape[-2], m.shape[-1] + for dr in range(3): + r_end = dr + h + for dc in range(3): + c_end = dc + w + out |= p[..., dr:r_end, dc:c_end] + return out + + import cv2 + + out = np.empty_like(mask, dtype=bool) + for idx in np.ndindex(mask.shape[:-2]): + out[idx] = cv2.dilate( + mask[idx].astype(np.uint8, copy=False), + kernel, + iterations=iterations, + ).astype(bool, copy=False) + return out diff --git a/src/giant_python/bandsilo/peaks.py b/src/giant_python/numerics/peaks.py similarity index 84% rename from src/giant_python/bandsilo/peaks.py rename to src/giant_python/numerics/peaks.py index 1fda7d6..cd27660 100644 --- a/src/giant_python/bandsilo/peaks.py +++ b/src/giant_python/numerics/peaks.py @@ -1,11 +1,11 @@ -"""Gaussian peak detection in the activity image. +"""Shared Gaussian peak detection in the activity image. Ported from the peak-detection kernels of ``extractSLAP2IntegrationSources.py`` (ref 307-712, originally ``getActImPeaks.m``). :func:`get_act_im_peaks` finds source seeds in the 3-D (Z, H, W) activity image produced by -:mod:`giant_python.bandsilo.summary_images`; those seeds initialize the Phase-6 -NMF localization. +:mod:`giant_python.extraction.band.summary_images`; those seeds initialize the +Phase-6 NMF localization. Detection fits integrated axis-aligned 2-D Gaussians (each Gaussian integrated over unit pixels) to the activity image with a bounded Levenberg-Marquardt @@ -23,7 +23,7 @@ from __future__ import annotations -from typing import Tuple +from typing import Tuple, cast import numpy as np import scipy.ndimage as ndimage @@ -37,6 +37,16 @@ _MAD_TO_SIGMA = 0.67449 +def _validate_theta(theta: np.ndarray) -> None: + """Require explicit amplitude, center and both axis-width columns.""" + if theta.ndim != 2 or theta.shape[1] != 5: + raise ValueError( + "Gaussian parameters must have shape (N, 5): " + "[amp, mu_y, mu_x, sigma_y, sigma_x]; " + "repeat the width for equal-width Gaussians" + ) + + def gaussian_peaks_integrated( theta: np.ndarray, yxdata: np.ndarray ) -> np.ndarray: @@ -48,9 +58,9 @@ def gaussian_peaks_integrated( Parameters ---------- - theta : ndarray of shape (N, 5) or (N, 4) - Per-Gaussian ``[amp, mu_y, mu_x, sigma_y, sigma_x]``. A four-column - array uses a shared ``sigma`` for both axes (legacy isotropic form). + theta : ndarray of shape (N, 5) + Per-Gaussian ``[amp, mu_y, mu_x, sigma_y, sigma_x]``. Equal widths + must be supplied explicitly in both width columns. yxdata : ndarray of shape (M, 2) Pixel-center ``[y, x]`` coordinates. @@ -59,6 +69,7 @@ def gaussian_peaks_integrated( ndarray of shape (M,) Predicted value at each pixel: ``sum_n A_n * Iy[y, n] * Ix[x, n]``. """ + _validate_theta(theta) x_int = yxdata[:, 1].astype(np.intp) y_int = yxdata[:, 0].astype(np.intp) x_min = int(x_int.min()) @@ -72,7 +83,7 @@ def gaussian_peaks_integrated( my = theta[:, 1] mx = theta[:, 2] sy = np.maximum(theta[:, 3], np.finfo(float).eps) - sx = np.maximum(theta[:, -1], np.finfo(float).eps) + sx = np.maximum(theta[:, 4], np.finfo(float).eps) c = np.sqrt(np.pi / 2) rt2 = np.sqrt(2.0) @@ -115,9 +126,8 @@ def _gaussian_peaks_integrated_val_jac( Parameters ---------- - theta : ndarray of shape (N, 5) or (N, 4) - Per-Gaussian ``[amp, mu_y, mu_x, sigma_y, sigma_x]``, or the legacy - isotropic form ``[amp, mu_y, mu_x, sigma]``. + theta : ndarray of shape (N, 5) + Per-Gaussian ``[amp, mu_y, mu_x, sigma_y, sigma_x]``. yxdata : ndarray of shape (M, 2) Pixel-center ``[y, x]`` coordinates. @@ -125,10 +135,10 @@ def _gaussian_peaks_integrated_val_jac( ------- val : ndarray of shape (M,) Forward value at each pixel. - jac : ndarray of shape (M, P*N) - Derivatives in the same order as ``theta.ravel()``. The isotropic - form sums the two width derivatives into one shared-sigma derivative. + jac : ndarray of shape (M, 5*N) + Derivatives in the same order as ``theta.ravel()``. """ + _validate_theta(theta) x_int = yxdata[:, 1].astype(np.intp) y_int = yxdata[:, 0].astype(np.intp) m = len(x_int) @@ -145,7 +155,7 @@ def _gaussian_peaks_integrated_val_jac( my = theta[:, 1] mx = theta[:, 2] sy = np.maximum(theta[:, 3], np.finfo(float).eps) - sx = np.maximum(theta[:, -1], np.finfo(float).eps) + sx = np.maximum(theta[:, 4], np.finfo(float).eps) c = np.sqrt(np.pi / 2) rt2 = np.sqrt(2.0) @@ -196,16 +206,12 @@ def _gaussian_peaks_integrated_val_jac( dval_dsy = amp[np.newaxis, :] * d_iy_ds_m * ix_m dval_dsx = amp[np.newaxis, :] * iy_m * d_ix_ds_m - n_params = theta.shape[1] - jac = np.empty((m, n_params * n), dtype=float) - jac[:, 0::n_params] = dval_da - jac[:, 1::n_params] = dval_dmy - jac[:, 2::n_params] = dval_dmx - if n_params == 4: - jac[:, 3::n_params] = dval_dsy + dval_dsx - else: - jac[:, 3::n_params] = dval_dsy - jac[:, 4::n_params] = dval_dsx + jac = np.empty((m, 5 * n), dtype=float) + jac[:, 0::5] = dval_da + jac[:, 1::5] = dval_dmy + jac[:, 2::5] = dval_dmx + jac[:, 3::5] = dval_dsy + jac[:, 4::5] = dval_dsx return val, jac @@ -217,6 +223,8 @@ def _lsq_curvefit( lb_flat: np.ndarray, ub_flat: np.ndarray, max_nfev: int = 5000, + *, + shared_widths: bool = False, ) -> np.ndarray: """Bounded Levenberg-Marquardt fit of integrated Gaussians. @@ -226,27 +234,39 @@ def _lsq_curvefit( Parameters ---------- - theta0 : ndarray of shape (N, P) - Initial ``[amp, mu_y, mu_x, sigma_y, sigma_x]`` per Gaussian (P=5), - or legacy isotropic parameters (P=4). + theta0 : ndarray of shape (N, 5) + Initial ``[amp, mu_y, mu_x, sigma_y, sigma_x]`` per Gaussian. xdata : ndarray of shape (M, 2) Pixel-center ``[y, x]`` coordinates. ydata : ndarray of shape (M,) Observed values (``act_im(sel) - mu_bg``). - lb_flat, ub_flat : ndarray of shape (P*N,) + lb_flat, ub_flat : ndarray of shape (5*N,) Row-major-flattened lower/upper bounds. max_nfev : int Cap on value+Jacobian evaluations. + shared_widths : bool + Constrain width updates to be equal for the isotropic warm start. + Both initial width columns and their bounds must already match. Returns ------- - ndarray of shape (N, P) + ndarray of shape (N, 5) Optimized parameters. """ - n_params = theta0.shape[1] + _validate_theta(theta0) + if lb_flat.shape != (theta0.size,) or ub_flat.shape != (theta0.size,): + raise ValueError("Gaussian bounds must have shape (5*N,)") + if shared_widths and ( + not np.array_equal(theta0[:, 3], theta0[:, 4]) + or not np.array_equal(lb_flat[3::5], lb_flat[4::5]) + or not np.array_equal(ub_flat[3::5], ub_flat[4::5]) + ): + raise ValueError( + "Shared-width fitting requires equal y/x widths and bounds" + ) x = np.clip(theta0.ravel().copy(), lb_flat, ub_flat) - val, jac = _gaussian_peaks_integrated_val_jac(x.reshape(-1, n_params), xdata) + val, jac = _gaussian_peaks_integrated_val_jac(x.reshape(-1, 5), xdata) r = val - ydata cost = np.dot(r, r) nfev = 1 @@ -257,15 +277,27 @@ def _lsq_curvefit( if nfev >= max_nfev: break - jtj = jac.T @ jac - jtr = jac.T @ r + if shared_widths: + # Optimize one width degree of freedom, keeping five-column + # parameters throughout. The chain rule ties the two widths. + full_jac = jac.reshape(len(r), -1, 5) + fit_jac = full_jac[:, :, :4].copy() + fit_jac[:, :, 3] = full_jac[:, :, 3] + full_jac[:, :, 4] + fit_jac = fit_jac.reshape(len(r), -1) + else: + fit_jac = jac + jtj = fit_jac.T @ fit_jac + jtr = fit_jac.T @ r diag_jtj = np.maximum(np.diag(jtj), 1e-8) delta = np.linalg.solve(jtj + lam * np.diag(diag_jtj), -jtr) + if shared_widths: + updates = delta.reshape(-1, 4) + delta = np.column_stack([updates, updates[:, 3]]).ravel() x_new = np.clip(x + delta, lb_flat, ub_flat) val_new, jac_new = _gaussian_peaks_integrated_val_jac( - x_new.reshape(-1, n_params), xdata + x_new.reshape(-1, 5), xdata ) r_new = val_new - ydata cost_new = np.dot(r_new, r_new) @@ -286,7 +318,7 @@ def _lsq_curvefit( else: lam = min(lam * 10.0, 1e10) - return x.reshape(-1, n_params) + return x.reshape(-1, 5) def _fit_isotropic_then_anisotropic( @@ -300,8 +332,9 @@ def _fit_isotropic_then_anisotropic( """Fit shared widths first, then relax to independent y/x widths. All peaks in the supplied fit region are optimized jointly in both stages. - The first stage uses four parameters per peak and constrains its shared - sigma to the intersection of the y/x bounds (normally [0.35, 1] pixels). + Both stages use five-column parameters. The first stage ties the two + widths and constrains their shared sigma to the intersection of the y/x + bounds (normally [0.35, 1] pixels). The second stage starts from that solution with sigma_y == sigma_x and restores the original five-parameter bounds. Each stage has its own ``max_nfev`` budget. @@ -311,28 +344,40 @@ def _fit_isotropic_then_anisotropic( solver accepts only improving steps, retaining the isotropic solution when relaxation cannot improve it. """ + _validate_theta(theta0) + if lb_flat.shape != (theta0.size,) or ub_flat.shape != (theta0.size,): + raise ValueError("Gaussian bounds must have shape (5*N,)") if theta0.shape[0] == 0: return theta0.copy() lb = lb_flat.reshape(-1, 5) ub = ub_flat.reshape(-1, 5) - iso_lb = lb[:, :4].copy() - iso_ub = ub[:, :4].copy() + iso_lb = lb.copy() + iso_ub = ub.copy() iso_lb[:, 3] = np.maximum(lb[:, 3], lb[:, 4]) iso_ub[:, 3] = np.minimum(ub[:, 3], ub[:, 4]) + iso_lb[:, 4] = iso_lb[:, 3] + iso_ub[:, 4] = iso_ub[:, 3] if np.any(iso_lb[:, 3] > iso_ub[:, 3]): - raise ValueError("Isotropic fitting requires overlapping y/x width bounds") + raise ValueError( + "Isotropic fitting requires overlapping y/x width bounds" + ) clipped = np.clip(theta0, lb, ub) - iso0 = clipped[:, :4].copy() + iso0 = clipped.copy() iso0[:, 3] = np.sqrt(clipped[:, 3] * clipped[:, 4]) + iso0[:, 4] = iso0[:, 3] isotropic = _lsq_curvefit( - iso0, xdata, ydata, iso_lb.ravel(), iso_ub.ravel(), + iso0, + xdata, + ydata, + iso_lb.ravel(), + iso_ub.ravel(), max_nfev=max_nfev, + shared_widths=True, ) - relaxed0 = np.column_stack([isotropic, isotropic[:, 3]]) return _lsq_curvefit( - relaxed0, xdata, ydata, lb_flat, ub_flat, max_nfev=max_nfev + isotropic, xdata, ydata, lb_flat, ub_flat, max_nfev=max_nfev ) @@ -399,7 +444,7 @@ def _buffer_mask(pim: np.ndarray, buffer_size: int) -> np.ndarray: def _process_cc_new_peak( - new_peak, + new_peak: tuple[tuple[int, int], int], thetaf: np.ndarray, p_locs: np.ndarray, labeled_full: np.ndarray, @@ -531,7 +576,8 @@ def _refine_residual_peaks( The refined Gaussian parameters. """ reject_mask = np.zeros((height, width), dtype=bool) - labeled_full, _ = ndimage.label(act_sel_pix) + # With output omitted, label returns (label array, component count). + labeled_full, _ = cast(tuple[np.ndarray, int], ndimage.label(act_sel_pix)) while True: e = res_im.copy() @@ -578,7 +624,9 @@ def _refine_residual_peaks( act_sel_pix = ndimage.binary_dilation(p_im, structure=np.ones((9, 9))) act_sel_pix &= ~np.isnan(act_im_2d) - labeled_full, _ = ndimage.label(act_sel_pix) + labeled_full, _ = cast( + tuple[np.ndarray, int], ndimage.label(act_sel_pix) + ) return thetaf @@ -642,8 +690,13 @@ def detect_peaks_2d( act_sel_pix &= ~np.isnan(act_im_2d) thetaf = np.column_stack( - [amp, py.astype(float), px.astype(float), - 0.5 * np.ones(n_peaks), 0.5 * np.ones(n_peaks)] + [ + amp, + py.astype(float), + px.astype(float), + 0.5 * np.ones(n_peaks), + 0.5 * np.ones(n_peaks), + ] ) p_locs = np.column_stack([py.astype(float), px.astype(float)]) @@ -685,7 +738,10 @@ def detect_peaks_2d( sy = thetaf[:, 3] sx = thetaf[:, 4] adj_thresh = peak_thresh / ( - np.pi / 2 * sy * sx + np.pi + / 2 + * sy + * sx * erf(1 / (np.sqrt(2) * sy)) * erf(1 / (np.sqrt(2) * sx)) ) @@ -697,7 +753,7 @@ def detect_peaks_2d( def get_act_im_peaks( act_im: np.ndarray, peak_th: float = 3.0, - exclusion_mask: np.ndarray = None, + exclusion_mask: np.ndarray | None = None, buffer_size: int = 0, ) -> np.ndarray: """Find Gaussian source seeds in a 3-D (Z, H, W) activity image. diff --git a/src/giant_python/parallel.py b/src/giant_python/parallel.py index 4c0fe45..12915f5 100644 --- a/src/giant_python/parallel.py +++ b/src/giant_python/parallel.py @@ -5,7 +5,7 @@ trials without each stage reimplementing the plumbing. """ -from typing import Callable, Iterable, List, Optional, TypeVar +from typing import Callable, Iterable, List, Optional, TypeVar, cast T = TypeVar("T") R = TypeVar("R") @@ -59,8 +59,11 @@ def map_trials( # trial *completes*, so wrapping this generator in tqdm tracks real # progress. Wrapping the dispatch generator instead would fill the bar as # soon as tasks are submitted, long before the workers finish. - results = Parallel(n_jobs=n_workers, return_as="generator")( - delayed(func)(t) for t in trials + results = cast( + Iterable[R], + Parallel(n_jobs=n_workers, return_as="generator")( + delayed(func)(t) for t in trials + ), ) if desc is not None: from tqdm import tqdm diff --git a/src/giant_python/pipeline/__init__.py b/src/giant_python/pipeline/__init__.py index 7bebdbc..28c68ce 100644 --- a/src/giant_python/pipeline/__init__.py +++ b/src/giant_python/pipeline/__init__.py @@ -1,53 +1,12 @@ -"""Pipeline orchestration for GIAnT-Python. +"""Implemented band annotation/extraction services and a chainable facade.""" -Thin stage classes (the "C" layer) that compose the functional/dataclass core -into the three pipeline stages, plus the :class:`Pipeline` facade that owns the -shared session state and exposes ``run_all``. The underlying stage *functions* -are also exported for power users who want to skip the classes. -""" - -from .base import Stage -from .extract import ( - BandSourceExtractor, - SourceExtractor, - StandardSourceExtractor, - extract_band_sources, - silo, -) -from .organize import ( - TrialTableBuilder, - build_trial_table, - build_trial_table_slap2, - verify_files, -) +from .annotate import annotate_rois +from .extract import extract_band_sources, extract_sources from .pipeline import Pipeline -from .register import ( - BandRegistration, - MotionCorrector, - MultiRoiRegistration, - StripRegistration, - band_registration, - multi_roi_registration, - strip_registration, -) __all__ = [ - "band_registration", - "BandRegistration", - "BandSourceExtractor", - "build_trial_table", - "build_trial_table_slap2", + "annotate_rois", "extract_band_sources", - "MotionCorrector", - "MultiRoiRegistration", - "multi_roi_registration", + "extract_sources", "Pipeline", - "silo", - "SourceExtractor", - "StandardSourceExtractor", - "Stage", - "StripRegistration", - "strip_registration", - "TrialTableBuilder", - "verify_files", ] diff --git a/src/giant_python/pipeline/annotate.py b/src/giant_python/pipeline/annotate.py new file mode 100644 index 0000000..48d88ea --- /dev/null +++ b/src/giant_python/pipeline/annotate.py @@ -0,0 +1,55 @@ +"""Public ROI annotation service with lazy band/GUI imports.""" + +from __future__ import annotations + +from typing import Optional + +from ..models.params import ( + AnnotationInput, + BandParamsInput, + ExecutionInput, +) +from .extract import TrialTableInput, _require_band_mode + + +def annotate_rois( + input: TrialTableInput, + params: BandParamsInput = None, + *, + microscope: str = "slap2", + scan_mode: Optional[str] = "band", + execution: ExecutionInput = None, + annotations: AnnotationInput = None, +) -> str: + """Load or draw and save band ROIs through the shared annotation service. + + Parameters + ---------- + input : str, Path or TrialTable + Exact filename or loaded trial table; never reconstructed from a + results directory. + params : BandSiloParams, dict or None + Scientific configuration only. + microscope : str + Currently only ``'slap2'`` is supported. + scan_mode : str or None + Same routing policy as ``extract_sources``. + execution : ExecutionOptions, dict or None + Run policy, including logging. + annotations : AnnotationOptions, dict or None + GUI policy and operator metadata. Calling this explicit annotation + step enables ROI handling for this call even if ``enabled=False``. + + Returns + ------- + str + Path to the written or existing annotation file. + """ + _require_band_mode(microscope, scan_mode) + if input is None: + raise ValueError("A trial-table path or TrialTable is required") + from ..extraction.band.annotation import annotate_band_rois + + return annotate_band_rois( + input, params, execution=execution, annotations=annotations + ) diff --git a/src/giant_python/pipeline/base.py b/src/giant_python/pipeline/base.py deleted file mode 100644 index dffb4ba..0000000 --- a/src/giant_python/pipeline/base.py +++ /dev/null @@ -1,23 +0,0 @@ -"""Abstract base class for pipeline stages.""" - -from __future__ import annotations - -from abc import ABC, abstractmethod -from typing import Any - - -class Stage(ABC): - """Base class for a single pipeline stage. - - A stage bundles its parameters and exposes a single :meth:`run` method. - Concrete stages (``TrialTableBuilder``, ``MotionCorrector``, - ``SourceExtractor``) delegate the actual computation to the functional - core and the math kernels. Stages are independently usable for partial - re-runs; the :class:`~giant_python.pipeline.pipeline.Pipeline` facade - composes them for the common end-to-end case. - """ - - @abstractmethod - def run(self, *args: Any, **kwargs: Any) -> Any: - """Execute the stage and return its result.""" - raise NotImplementedError # pragma: no cover - abstract stub diff --git a/src/giant_python/pipeline/extract.py b/src/giant_python/pipeline/extract.py index f94b8d4..ef2f5fe 100644 --- a/src/giant_python/pipeline/extract.py +++ b/src/giant_python/pipeline/extract.py @@ -1,16 +1,7 @@ -"""Stage 3 - source extraction / SILo (port of SILo.m). - -Exposes the porting-faithful extraction functions plus a -:class:`SourceExtractor` base with one backend per SLAP2 scan mode -(standard pixel-movie SILo / band-scan SILo), selected via -:meth:`SourceExtractor.for_scan_mode` -- mirroring the -:class:`~giant_python.pipeline.register.MotionCorrector` backend pattern. - -Both backends share the same algorithmic skeleton (activity image -> peak -detection -> source localization -> trace extraction -> baseline/dF -> -deconvolution); the shared steps live as pure kernels in -:mod:`giant_python.math` (``peaks``, ``baseline``, ``deconv``) so that only -the scan-mode-specific data loading and geometry differ between backends. +"""Canonical standalone extraction services. + +Only SLAP2 band extraction is supported. Backend imports are lazy; no input +table is reloaded here. """ from __future__ import annotations @@ -18,151 +9,111 @@ from pathlib import Path from typing import Optional, Union -from ..models.params import SiloParams -from .base import Stage +from ..models.experiment_summary import ExperimentSummary +from ..models.params import ( + AnnotationInput, + BandParamsInput, + ExecutionInput, +) +from ..models.trial_table import TrialTable +TrialTableInput = Union[str, Path, TrialTable] -def silo( - dr_or_path_to_trial_table: Optional[Union[str, Path]] = None, - params_in: Optional[dict] = None, -) -> None: - """Perform standard (pixel-movie) source extraction and localization. - Detects active synaptic boutons or spines from registered movies using - peak-finding on a per-trial activity image followed by NMF-based trace - refinement across trials. Results are saved as an experiment summary - file. Corresponds to SILo.m in GIAnT-MATLAB. +def _require_band_mode( + microscope: str, + scan_mode: Optional[str], +) -> None: + """Reject unsupported routing before loading data or a backend.""" + if microscope != "slap2": + raise ValueError( + "Only microscope='slap2', scan_mode='band' is supported" + ) + mode = "band" if scan_mode is None else scan_mode + if mode != "band": + raise ValueError( + f"Source extraction for scan_mode={mode!r} is unsupported; " + "only 'band' is supported" + ) + + +def extract_sources( + input: TrialTableInput, + params: BandParamsInput = None, + *, + microscope: str = "slap2", + scan_mode: Optional[str] = "band", + execution: ExecutionInput = None, + annotations: AnnotationInput = None, +) -> ExperimentSummary: + """Extract sources through the supported workflow service. Parameters ---------- - dr_or_path_to_trial_table : str or Path, optional - Either a directory containing a trial table file, or the full path - to a trial table file. - params_in : dict, optional - Parameter overrides. Keys and default values are defined in - :func:`giant_python.models.params.set_params` under ``"SILo"``. + input : str, Path or TrialTable + Exact trial-table filename or a loaded table, passed through intact. + params : BandSiloParams, dict or None + Scientific settings only. + microscope : str + Currently only ``'slap2'`` is supported. + scan_mode : str or None + Explicit routing, defaulting to band. ``None`` also selects band. + execution : ExecutionOptions, dict or None + Run policy, independent of science. + annotations : AnnotationOptions, dict or None + ROI inclusion and interactivity policy. + + Returns + ------- + ExperimentSummary + Eager extraction results, also persisted by the backend. + + Raises + ------ + ValueError + If an unsupported microscope or scan mode is requested. """ - raise NotImplementedError + _require_band_mode(microscope, scan_mode) + return extract_band_sources( + input, params, execution=execution, annotations=annotations + ) def extract_band_sources( - path_to_trial_table: Optional[Union[str, Path]] = None, - params_in: Optional[Union[dict, SiloParams]] = None, -) -> "object": + input: TrialTableInput, + params: BandParamsInput = None, + *, + execution: ExecutionInput = None, + annotations: AnnotationInput = None, +) -> ExperimentSummary: """Perform band-scan source extraction for SLAP2. The band-scan variant of SILo: operates on superpixel / DMD-geometry - data and the band lookup table produced by - :func:`giant_python.pipeline.register.band_registration` + data and the externally produced band registration lookup table (``fnAdataInt``), rather than on a reconstructed pixel movie. Corresponds to ``extractSLAP2IntegrationSources.py`` in ophys-slap2-analysis. - A thin adapter over :func:`giant_python.bandsilo.pipeline\ -.extract_band_sources` (imported lazily so ``import giant_python`` never - pulls in the heavy BandSILo dependencies). + Delegates lazily to ``giant_python.extraction.band.workflow``. All option + resolution belongs to that service. Parameters ---------- - path_to_trial_table : str or Path, optional - Full path to the ``trial_table.h5`` file (with band alignment - data). - params_in : dict or SiloParams, optional - Parameter overrides. + input : str, Path or TrialTable + Trial-table path or loaded metadata with band alignment data. + params : BandSiloParams, dict or None + Scientific settings only. + execution : ExecutionOptions, dict or None + Run policy, independent of scientific settings. + annotations : AnnotationOptions, dict or None + ROI inclusion and GUI policy. Returns ------- ExperimentSummary The extracted sources and summary (also written to disk). """ - from ..bandsilo.pipeline import extract_band_sources as _extract_impl - - return _extract_impl(path_to_trial_table, params_in) - - -class SourceExtractor(Stage): - """Stage 3 base class: extract sources from motion-corrected trials. - - Use :meth:`for_scan_mode` to obtain the appropriate backend. Concrete - subclasses wrap the corresponding extraction function. - - Parameters - ---------- - params : SiloParams, optional - Source-extraction parameters; defaults to :class:`SiloParams` - defaults. - """ - - def __init__(self, params: Optional[SiloParams] = None) -> None: - """Store the extraction parameters (default :class:`SiloParams`).""" - self.params = params or SiloParams() - - @staticmethod - def for_scan_mode( - scan_mode: str, - params: Optional[SiloParams] = None, - ) -> "SourceExtractor": - """Return the extraction backend for the given SLAP2 scan mode. - - Parameters - ---------- - scan_mode : str - ``"standard"`` (-> :class:`StandardSourceExtractor`) or - ``"band"`` (-> :class:`BandSourceExtractor`). - params : SiloParams, optional - Source-extraction parameters. - - Returns - ------- - SourceExtractor - The appropriate backend instance. - - Raises - ------ - ValueError - If *scan_mode* is not ``"standard"`` or ``"band"``. - """ - if scan_mode == "band": - return BandSourceExtractor(params) - if scan_mode == "standard": - return StandardSourceExtractor(params) - raise ValueError(f"Unknown scan mode: {scan_mode!r}") - - def run(self, trial_table: "object") -> "object": - """Extract sources and write experiment_summary.h5. - - Parameters - ---------- - trial_table : TrialTable - A trial table whose trials have been motion-corrected. - - Returns - ------- - ExperimentSummary - The extracted sources and summary (also persisted to disk). - """ - raise NotImplementedError - - -class StandardSourceExtractor(SourceExtractor): - """Standard pixel-movie SILo backend (wraps :func:`silo`).""" - - def run(self, trial_table: "object") -> "object": - """Extract from pixel movies. See :meth:`SourceExtractor.run`.""" - raise NotImplementedError - - -class BandSourceExtractor(SourceExtractor): - """Band-scan SILo backend. - - Wraps :func:`extract_band_sources`. - """ - - def run(self, trial_table: "object") -> "object": - """Extract from band data. See :meth:`SourceExtractor.run`. + if input is None: + raise ValueError("A trial-table path or TrialTable is required") + from ..extraction.band.workflow import extract_band_sources as extract - Resolves the on-disk ``trial_table.h5`` path from the trial table's - ``savedr`` and delegates to :func:`extract_band_sources` with - this extractor's parameters. - """ - path = Path(trial_table.savedr) / "trial_table.h5" - return extract_band_sources(path, self.params) + return extract(input, params, execution=execution, annotations=annotations) diff --git a/src/giant_python/pipeline/organize.py b/src/giant_python/pipeline/organize.py deleted file mode 100644 index 97d6492..0000000 --- a/src/giant_python/pipeline/organize.py +++ /dev/null @@ -1,142 +0,0 @@ -"""Stage 1 - trial organization (ports of buildTrialTable[SLAP2].m). - -Builds the trial table that threads through the rest of the pipeline, plus the -``verify_files`` integrity check. Exposes both the porting-faithful functions -and the :class:`TrialTableBuilder` stage class. -""" - -from __future__ import annotations - -from pathlib import Path -from typing import Optional, Tuple, Union - -import numpy as np - -from .base import Stage - - -def build_trial_table( - dr: Optional[Union[str, Path]] = None, - fns: Optional[Union[str, list]] = None, - save_dr: Optional[Union[str, Path]] = None, -) -> dict: - """Organize multi-trial recordings and metadata for Bergamo recordings. - - Groups TIFF or HDF5 recording files into epochs and trials, populating - a trial table that is saved to disk. Corresponds to buildTrialTable.m - in GIAnT-MATLAB. - - Parameters - ---------- - dr : str or Path, optional - Directory containing TIFF or HDF5 recording files. - fns : str or list, optional - Filename or list of filenames to include. Pass ``True`` to - auto-select all non-registered TIFF files in *dr*. - save_dr : str or Path, optional - Directory in which to save the trial table. Defaults to *dr*. - - Returns - ------- - dict - Trial table with keys ``filename``, ``trueTrialIx``, ``epoch``, - ``trialEndTimeFromPC``, and ``trialStartTimeInferred``. - """ - raise NotImplementedError - - -def build_trial_table_slap2( - dr: Optional[Union[str, Path]] = None, - save_dr: Optional[Union[str, Path]] = None, -) -> dict: - """Organize multi-trial recordings for the SLAP2 data processing pipeline. - - Groups DMD1 and DMD2 DAT files into epochs and trials, loads reference - stacks, and populates a trial table saved to disk. Corresponds to - buildTrialTableSLAP2.m in GIAnT-MATLAB. - - Parameters - ---------- - dr : str or Path, optional - Directory containing DAT recording files and reference TIFF stacks. - save_dr : str or Path, optional - Directory in which to save the trial table. Defaults to *dr*. - - Returns - ------- - dict - Trial table with keys ``filename``, ``firstLine``, ``lastLine``, - ``trueTrialIx``, ``epoch``, ``refStack``, ``trialEndTimeFromPC``, - ``trialStartTimeInferred``, ``datadr``, and ``savedr``. - """ - raise NotImplementedError - - -def verify_files( - fn: Union[str, Path], - dr: Union[str, Path], - params: dict, -) -> Tuple[dict, np.ndarray]: - """Verify that all files referenced by a trial table exist on disk. - - Loads the trial table and checks for the registered TIFF or HDF5 movies, - alignment data files, and raw source data files for each trial and DMD. - Returns the updated trial table and a boolean keep-mask. Corresponds to - verifyFiles.m in GIAnT-MATLAB. - - Parameters - ---------- - fn : str or Path - Filename of the trial table file (relative to *dr*). - dr : str or Path - Directory containing the trial table and associated data. - params : dict - Parameter dictionary, as returned by - :func:`giant_python.models.params.set_params`. - - Returns - ------- - trial_table : dict - Loaded (and possibly path-corrected) trial table. - keep_trials : ndarray of bool, shape (n_dmds, n_trials) - ``True`` for each trial that has all required files present. - """ - raise NotImplementedError - - -class TrialTableBuilder(Stage): - """Stage 1: build the trial table for a microscope's raw data. - - Dispatches to :func:`build_trial_table` (Bergamo) or - :func:`build_trial_table_slap2` (SLAP2) based on *microscope*. - - Parameters - ---------- - microscope : str - ``"slap2"`` or ``"bergamo"``. - """ - - def __init__(self, microscope: str = "slap2") -> None: - """Store the microscope kind (``"slap2"`` or ``"bergamo"``).""" - self.microscope = microscope - - def run( - self, - data_dir: Union[str, Path], - save_dir: Union[str, Path], - ) -> "object": - """Organize trials and write trial_table.h5. - - Parameters - ---------- - data_dir : str or Path - Directory containing the raw recording files. - save_dir : str or Path - Directory in which to write trial_table.h5. - - Returns - ------- - TrialTable - The organized trial table (also persisted to disk). - """ - raise NotImplementedError diff --git a/src/giant_python/pipeline/pipeline.py b/src/giant_python/pipeline/pipeline.py index a5ec2e2..ac5751b 100644 --- a/src/giant_python/pipeline/pipeline.py +++ b/src/giant_python/pipeline/pipeline.py @@ -1,42 +1,43 @@ -"""The :class:`Pipeline` facade: the full-pipeline orchestration object. - -This is a thin *coordinator*, deliberately not a god-object. It holds only -lightweight session state (config, paths, and the ``TrialTable`` metadata), -delegates all real work to the stage classes (which remain independently -usable for partial re-runs), and owns the cross-cutting concerns that belong -to no single stage: ``run_all``, resume/checkpointing, logging, and config -loading. - -Crucially it does **not** cache heavy pixel arrays, mean images, or source -matrices in memory; those stay on disk in HDF5/TIFF and are loaded lazily by -each stage. ``self.summary`` references an ``ExperimentSummary`` whose heavy -arrays are themselves disk-backed. +"""Chainable coordinator over the same services as standalone functions. + +Band annotation/extraction starts from existing registered trial metadata. +Returned summaries currently contain eager arrays. """ from __future__ import annotations +from dataclasses import replace from pathlib import Path -from typing import Optional, Union +from typing import Optional -from ..models.params import AlignParams, SiloParams -from .extract import SourceExtractor -from .organize import TrialTableBuilder -from .register import MotionCorrector +from ..models.experiment_summary import ExperimentSummary +from ..models.params import ( + AnnotationInput, + BandParamsInput, + BandSiloParams, + ExecutionInput, + resolve_band_options, +) +from ..models.trial_table import TrialTable +from .annotate import annotate_rois +from .extract import TrialTableInput, extract_sources class Pipeline: - """Coordinator for the full organize -> register -> extract pipeline. + """Coordinator for annotation and extraction from a registered trial table. Parameters ---------- microscope : str - ``"slap2"`` or ``"bergamo"``. - save_dir : str or Path - Directory in which all results are written. - align_params : AlignParams, optional - Motion-correction parameters. - silo_params : SiloParams, optional - Source-extraction parameters. + Acquisition type; only ``"slap2"`` band extraction is implemented. + params : BandSiloParams, dict or None + Scientific settings only; defaults to band. + execution : ExecutionOptions, dict or None + Run policy, separate from science. + annotations : AnnotationOptions, dict or None + ROI inclusion and GUI policy. + scan_mode : str or None + Optional explicit routing override; unsupported modes fail honestly. Attributes ---------- @@ -44,167 +45,123 @@ class Pipeline: The session's trial table (lightweight metadata), threaded between stages. summary : ExperimentSummary or None - The extraction result (heavy arrays disk-backed). + The extraction result (eager arrays, not a disk-backed proxy). + trial_table_path : Path or None + Exact source filename, when known. Never inferred from ``savedr``. """ def __init__( self, - microscope: str, - save_dir: Union[str, Path], - align_params: Optional[AlignParams] = None, - silo_params: Optional[SiloParams] = None, + microscope: str = "slap2", + params: BandParamsInput = None, + *, + execution: ExecutionInput = None, + annotations: AnnotationInput = None, + scan_mode: Optional[str] = "band", ) -> None: """Store session config; ``tt``/``summary`` start empty.""" self.microscope = microscope - self.save_dir = Path(save_dir) - self.align_params = align_params or AlignParams() - self.silo_params = silo_params or SiloParams(microscope=microscope) - self.tt: Optional[object] = None - self.summary: Optional[object] = None + self.params = BandSiloParams() if params is None else params + self.execution = execution + self.annotations = annotations + self.scan_mode = scan_mode + self.tt: Optional[TrialTable] = None + self.summary: Optional[ExperimentSummary] = None + self.trial_table_path: Optional[Path] = None + self.annotation_path: Optional[str] = None @classmethod def from_trial_table( cls, - path: Union[str, Path], - microscope: str, - **kwargs: object, + path: TrialTableInput, + microscope: str = "slap2", + **kwargs, ) -> "Pipeline": - """Resume a session from an existing trial_table.h5. + """Start a session from a trial-table path or loaded metadata. Parameters ---------- - path : str or Path - Path to an existing trial_table.h5 file. + path : str, Path or TrialTable + Exact filename or an already loaded table. A path is read once; + in-memory inputs are not copied, serialized or reloaded. microscope : str - ``"slap2"`` or ``"bergamo"``. + ``"slap2"``; other acquisition types are unsupported. **kwargs - Forwarded to :class:`Pipeline` (e.g. ``align_params``). - - Returns - ------- - Pipeline - A pipeline with ``tt`` loaded from disk. - """ - raise NotImplementedError - - @classmethod - def from_config(cls, path: Union[str, Path]) -> "Pipeline": - """Construct a pipeline from a declarative config file. - - Hook for the future config-driven (YAML/TOML) run mode. - - Parameters - ---------- - path : str or Path - Path to the config file. + Forwarded to :class:`Pipeline` (e.g. ``params`` or ``execution``). Returns ------- Pipeline - A configured pipeline. + A pipeline retaining ``tt`` and its known source filename. """ - raise NotImplementedError - - def organize(self, data_dir: Union[str, Path]) -> "Pipeline": - """Run stage 1 (trial organization), storing the result in ``self.tt``. - - Parameters - ---------- - data_dir : str or Path - Directory containing the raw recording files. - - Returns - ------- - Pipeline - ``self`` (supports method chaining). - """ - self.tt = TrialTableBuilder(self.microscope).run( - data_dir, self.save_dir - ) - return self - - def register(self) -> "Pipeline": - """Run stage 2 (motion correction), updating ``self.tt``. - - Returns - ------- - Pipeline - ``self`` (supports method chaining). - """ - corrector = MotionCorrector.for_microscope( - self.microscope, self.align_params + if isinstance(path, TrialTable): + table = path + source_path = table.source_path + elif isinstance(path, (str, Path)): + source_path = Path(path) + table = TrialTable.from_h5(source_path) + # Keep the supplied filename even when savedr points elsewhere. + table.source_path = source_path + else: + raise TypeError("Expected a trial-table path or TrialTable") + pipeline = cls(microscope=microscope, **kwargs) + pipeline.tt = table + pipeline.trial_table_path = ( + Path(source_path) if source_path is not None else None ) - self.tt = corrector.run(self.tt) - return self + return pipeline + + def _require_trial_table(self) -> TrialTable: + """Return loaded metadata or explain how to initialize a session.""" + if self.tt is None: + raise ValueError( + "No trial table is loaded; use Pipeline.from_trial_table() " + "before annotation or extraction" + ) + return self.tt def annotate(self) -> "Pipeline": - """Run the optional ROI-annotation GUI step. + """Load or draw band ROIs, retaining the same trial-table object. - Dispatches on the source-extraction scan mode: ``"band"`` uses the - BandSILo standalone annotation step (writing - ``/annotations/annotations.h5``); any other mode uses the - generic annotation GUI. GUI modules are imported lazily so headless - installs never import the GUI extra. + Uses the public annotation service and enables ROI inclusion for a + subsequent ``extract()`` after success. Drawing is still subject to + explicit/headless/automatic GUI policy. Caller options are not edited. Returns ------- Pipeline ``self`` (supports method chaining). """ - if self.silo_params.scan_mode == "band": - from ..bandsilo.annotate import annotate_band_rois - - annotate_band_rois( - Path(self.tt.savedr) / "trial_table.h5", self.silo_params - ) - else: - from ..gui import annotate_rois - - annotate_rois(self.tt) + table = self._require_trial_table() + self.annotation_path = annotate_rois( + table, + self.params, + microscope=self.microscope, + scan_mode=self.scan_mode, + execution=self.execution, + annotations=self.annotations, + ) + _, _, policy = resolve_band_options( + self.params, self.execution, self.annotations + ) + self.annotations = replace(policy, enabled=True) return self def extract(self) -> "Pipeline": - """Run stage 3 (source extraction), storing ``self.summary``. + """Run source extraction, storing ``self.summary``. Returns ------- Pipeline ``self`` (supports method chaining). """ - extractor = SourceExtractor.for_scan_mode( - self.silo_params.scan_mode, self.silo_params + table = self._require_trial_table() + self.summary = extract_sources( + table, + self.params, + microscope=self.microscope, + scan_mode=self.scan_mode, + execution=self.execution, + annotations=self.annotations, ) - self.summary = extractor.run(self.tt) return self - - def run_all( - self, - data_dir: Union[str, Path], - *, - annotate: bool = False, - resume: bool = True, - ) -> "Pipeline": - """Run the whole pipeline end-to-end. - - Parameters - ---------- - data_dir : str or Path - Directory containing the raw recording files. - annotate : bool - Whether to run the GUI annotation step between registration and - extraction. - resume : bool - Skip stages whose HDF5 outputs already exist (mirrors the - ``overwriteExisting`` logic in GIAnT-MATLAB). Reserved for the - per-stage checkpoint logic; not yet honored. - - Returns - ------- - Pipeline - ``self`` (with ``tt`` and ``summary`` populated). - """ - del resume # reserved for future per-stage checkpointing - self.organize(data_dir).register() - if annotate: - self.annotate() - return self.extract() diff --git a/src/giant_python/pipeline/register.py b/src/giant_python/pipeline/register.py deleted file mode 100644 index 4c865c3..0000000 --- a/src/giant_python/pipeline/register.py +++ /dev/null @@ -1,173 +0,0 @@ -"""Stage 2 - motion correction (ports of *Registration.m). - -Exposes the porting-faithful registration functions plus a -:class:`MotionCorrector` ABC with one backend per MATLAB registration variant -(MultiRoi / Strip / Band). Band-scan registration is exposed as -:func:`band_registration` / :class:`BandRegistration` (the -Python API name) and corresponds to ``BandRegistration.m``, writing -``bandRegLookupTable.h5``. -""" - -from __future__ import annotations - -from pathlib import Path -from typing import Optional, Union - -from ..models.params import AlignParams -from .base import Stage - - -def multi_roi_registration( - full_path_to_trial_table: Optional[Union[str, Path]] = None, - params_in: Optional[dict] = None, -) -> None: - """Perform motion correction for SLAP2 multi-ROI recordings. - - Registers each trial to a common template computed from initial frames, - producing downsampled registered TIFF movies and alignment metadata for - each DMD. Corresponds to MultiRoiRegistration.m in GIAnT-MATLAB. - - Parameters - ---------- - full_path_to_trial_table : str or Path, optional - Full path to the trial table file. - params_in : dict, optional - Parameter overrides. Keys and default values are defined in - :func:`giant_python.models.params.set_params` under - ``"MultiRoiRegistration"``. - """ - raise NotImplementedError - - -def strip_registration( - dr: Optional[Union[str, Path]] = None, - fns: Optional[Union[str, list]] = None, - params_in: Optional[dict] = None, -) -> None: - """Perform strip-based motion correction for Bergamo recordings. - - Registers each trial using DFT-based rigid registration, producing - downsampled and full-resolution registered TIFF or HDF5 movies together - with per-trial alignment metadata. Corresponds to StripRegistration.m in - GIAnT-MATLAB. - - Parameters - ---------- - dr : str or Path, optional - Directory containing TIFF or HDF5 files, or the path to a trial - table file. - fns : str or list, optional - Filename(s) to register, or path to a trial table file. - params_in : dict, optional - Parameter overrides. Keys and default values are defined in - :func:`giant_python.models.params.set_params` under - ``"StripRegistration"``. - """ - raise NotImplementedError - - -def band_registration( - full_path_to_trial_table: Optional[Union[str, Path]] = None, - params_in: Optional[dict] = None, -) -> None: - """Perform band-scan motion correction for SLAP2 recordings. - - Builds the cached ``motion_correction/bandRegLookupTable.h5`` lookup table - (on first run) and writes per-trial ``*_ALIGNMENTDATA.h5`` files including - BandRegistration-only fields ``brightnessDS`` and ``logLikelihoodDS``. - Corresponds to ``BandRegistration.m`` in GIAnT-MATLAB. - - Parameters - ---------- - full_path_to_trial_table : str or Path, optional - Full path to the trial table file. - params_in : dict, optional - Parameter overrides. Keys and default values are defined in - :func:`giant_python.models.params.set_params` under - ``"BandRegistration"``. - """ - raise NotImplementedError - - -class MotionCorrector(Stage): - """Stage 2 base class: register all trials in a trial table. - - Use :meth:`for_microscope` to obtain the appropriate backend. Concrete - subclasses wrap the corresponding registration function. - - Parameters - ---------- - params : AlignParams, optional - Alignment parameters; defaults to :class:`AlignParams` defaults. - """ - - def __init__(self, params: Optional[AlignParams] = None) -> None: - """Store the alignment parameters (default :class:`AlignParams`).""" - self.params = params or AlignParams() - - @staticmethod - def for_microscope( - microscope: str, - params: Optional[AlignParams] = None, - ) -> "MotionCorrector": - """Return the registration backend for the given microscope. - - Parameters - ---------- - microscope : str - ``"slap2"`` (-> :class:`MultiRoiRegistration`) or ``"bergamo"`` - (-> :class:`StripRegistration`). - params : AlignParams, optional - Alignment parameters. - - Returns - ------- - MotionCorrector - The appropriate backend instance. - """ - raise NotImplementedError - - def run(self, trial_table: "object") -> "object": - """Register every trial and update the trial table in place. - - Parameters - ---------- - trial_table : TrialTable - The organized trial table. - - Returns - ------- - TrialTable - The trial table with its ``motion_correction`` group populated. - """ - raise NotImplementedError - - -class MultiRoiRegistration(MotionCorrector): - """SLAP2 multi-ROI backend (wraps ``multi_roi_registration``).""" - - def run(self, trial_table: "object") -> "object": - """Register SLAP2 multi-ROI trials. See :meth:`MotionCorrector.run`.""" - raise NotImplementedError - - -class StripRegistration(MotionCorrector): - """Bergamo strip backend (wraps ``strip_registration``).""" - - def run(self, trial_table: "object") -> "object": - """Register Bergamo strip trials. See :meth:`MotionCorrector.run`.""" - raise NotImplementedError - - -class BandRegistration(MotionCorrector): - """SLAP2 BandRegistration backend (wraps ``band_registration``). - - Corresponds to ``BandRegistration.m``; writes ``bandRegLookupTable.h5``. - """ - - def run(self, trial_table: "object") -> "object": - """Register SLAP2 band trials. - - See ``MotionCorrector.run``. - """ - raise NotImplementedError diff --git a/src/giant_python/bandsilo/progress.py b/src/giant_python/progress.py similarity index 92% rename from src/giant_python/bandsilo/progress.py rename to src/giant_python/progress.py index 990d6ee..e61444b 100644 --- a/src/giant_python/bandsilo/progress.py +++ b/src/giant_python/progress.py @@ -1,9 +1,9 @@ -"""Opt-in verbose logging + progress bars for the band pipeline. +"""Shared opt-in verbose logging and progress bars. The BandSILo port stays quiet by default (headless / batch friendly); the reference's unconditional ``print``s and ``tqdm`` bars were dropped. These helpers restore that feedback opt-in: when ``verbose`` is set -(``SiloParams.verbose`` / ``giant ... --verbose``) stage messages are printed +(``ExecutionOptions.verbose`` / ``giant ... --verbose``) messages are printed and long-running loops show a progress bar. ``tqdm`` is imported lazily, so a non-verbose run neither prints nor imports it. """ diff --git a/tests/test_acquisition_boundaries.py b/tests/test_acquisition_boundaries.py new file mode 100644 index 0000000..f2921a8 --- /dev/null +++ b/tests/test_acquisition_boundaries.py @@ -0,0 +1,215 @@ +"""Guard the band acquisition boundary without changing numerical policy.""" + +import ast +import inspect +import tempfile +import unittest +from importlib import resources +from pathlib import Path +from unittest.mock import call, patch + +import h5py +import numpy as np +import tifffile + +from giant_python.extraction.band import annotation, geometry, inputs, workflow +from giant_python.io import slap2 +from giant_python.io.tiff import read_tiff + + +class TestAcquisitionOwnership(unittest.TestCase): + """File loading cannot leak back into geometry or duplicate raw IO APIs.""" + + def test_geometry_imports_only_numerical_support(self): + """Inspect all imports, including deferred imports inside functions.""" + tree = ast.parse(inspect.getsource(geometry)) + allowed = {"__future__", "typing", "numpy", "scipy"} + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + self.assertIn(alias.name.split(".")[0], allowed) + elif isinstance(node, ast.ImportFrom): + self.assertEqual(node.level, 0) + self.assertIn((node.module or "").split(".")[0], allowed) + elif isinstance(node, ast.Call) and isinstance( + node.func, ast.Name + ): + self.assertNotIn(node.func.id, {"open", "__import__"}) + + def test_loaders_have_one_owner(self): + """Workflows use input loaders, not numerical geometry or raw IO.""" + for name in ( + "load_lookup_table", + "_ref_stack_group", + "find_reference_file", + "load_reference_stack", + "default_psf", + "load_psf", + ): + with self.subTest(name=name): + function = getattr(inputs, name) + self.assertEqual(function.__module__, inputs.__name__) + self.assertFalse(hasattr(geometry, name)) + self.assertFalse(hasattr(slap2, name)) + self.assertIs(workflow.inputs, inputs) + self.assertIs(annotation.inputs, inputs) + self.assertIs(inputs.read_tiff, read_tiff) + self.assertIs(inputs.build_combined_psf, geometry.build_combined_psf) + self.assertIs( + inputs.threshold_and_crop_psf, geometry.threshold_and_crop_psf + ) + + def test_reader_protocol_method_is_docstring_only(self): + """The reader contract has no executable placeholder statement.""" + tree = ast.parse(inspect.getsource(slap2.Slap2Reader)) + method = next( + node + for node in tree.body[0].body + if isinstance(node, ast.FunctionDef) and node.name == "getLineData" + ) + self.assertEqual(len(method.body), 1) + self.assertIsNotNone(ast.get_docstring(method)) + self.assertIsNone(method.returns) + + +class TestBandInputLoading(unittest.TestCase): + """Preserve lookup layouts, reference page order and bundled PSF values.""" + + def test_lookup_aliases_layouts_and_int32_columns(self): + """Path groups take precedence; DMD groups and MATLAB layout work.""" + expected = { + "allSuperPixelIDs": np.array([[7], [3], [11]], dtype=np.int32), + "sparseMaskInds": np.array( + [[19, 1], [5, 2], [27, 3]], dtype=np.int32 + ), + "fastZ2RefZ": np.array([[4], [2]], dtype=np.int32), + } + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "lookup.h5" + for layout in (None, 0, 1): + with self.subTest(row_major=layout): + with h5py.File(path, "w") as file: + if layout is not None: + file.create_dataset("row_major", data=layout) + for group in ("Path1", "DMD2"): + for key, value in expected.items(): + file.create_dataset( + f"{group}/{key}", + data=(value if layout else value.T).astype( + np.float64 + ), + ) + file.create_group("DMD1") + result = inputs.load_lookup_table(path, 2) + self.assertEqual(set(result), set(expected)) + for key, value in expected.items(): + self.assertEqual(set(result[key]), {"DMD1", "DMD2"}) + for actual in result[key].values(): + self.assertEqual(actual.dtype, np.dtype("int32")) + np.testing.assert_array_equal(actual, value) + + def test_reference_tiff_values_shapes_and_dtypes(self): + """Decoding is raw; band scaling and channel reshape stay local.""" + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "sample_DMD2-REFERENCE.tif" + for dtype in (np.uint16, np.float32): + for group in ("Path2", "DMD2"): + with self.subTest(dtype=dtype, group=group): + raw = np.arange(120).reshape(6, 4, 5).astype(dtype) + if dtype == np.float32: + raw[1, 2, 3] = np.nan + tifffile.imwrite(path, raw, photometric="minisblack") + channels = np.array([[2], [1]], dtype=np.int16) + meta = {group: {"channels": channels}} + with patch.object( + inputs, "read_tiff", wraps=read_tiff + ) as reader: + stack, actual_channels, filename = ( + inputs.load_reference_stack(directory, meta, 1) + ) + reader.assert_called_once_with(str(path)) + decoded = read_tiff(path) + self.assertEqual(decoded.dtype, raw.dtype) + np.testing.assert_array_equal(decoded, raw) + expected = (raw / 100).reshape(3, 2, 4, 5) + expected = expected.transpose(1, 0, 2, 3) + self.assertEqual(stack.shape, (2, 3, 4, 5)) + self.assertEqual(stack.dtype, expected.dtype) + np.testing.assert_array_equal(stack, expected) + np.testing.assert_array_equal(actual_channels, [2, 1]) + self.assertEqual(actual_channels.dtype, channels.dtype) + np.testing.assert_array_equal(channels, [[2], [1]]) + self.assertEqual(filename, str(path)) + + def test_reference_discovery_precedence_and_missing(self): + """Recursive CONFIG2 wins; alternate and missing cases still work.""" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + fallback = root / "sample_DMD1-REFERENCE.tif" + preferred = root / "nested" / "sample_DMD1_CONFIG2-REFERENCE.tif" + preferred.parent.mkdir() + fallback.touch() + self.assertEqual( + inputs.find_reference_file(root, 0), str(fallback) + ) + preferred.touch() + self.assertEqual( + inputs.find_reference_file(root, 0), str(preferred) + ) + channels = np.array([[3, 1]], dtype=np.uint8) + with patch.object(inputs, "read_tiff") as reader: + stack, actual, filename = inputs.load_reference_stack( + root, {"DMD2": {"channels": channels}}, 1 + ) + reader.assert_not_called() + self.assertIsNone(stack) + self.assertIsNone(filename) + self.assertEqual(actual.dtype, channels.dtype) + np.testing.assert_array_equal(actual, [3, 1]) + + def test_reference_metadata_prefers_path_group(self): + """Retain subgroup identity and reject missing reference metadata.""" + primary = {"channels": [2, 1]} + fallback = {"channels": [3]} + meta = {"Path1": primary, "DMD1": fallback} + self.assertIs(inputs._ref_stack_group(meta, 0), primary) + self.assertIs(inputs._ref_stack_group({"DMD1": fallback}, 0), fallback) + with self.assertRaisesRegex( + ValueError, "Missing reference-stack group Path2 or DMD2" + ): + inputs._ref_stack_group(meta, 1) + + def test_bundled_psf_preserves_asset_values_and_per_dmd_cropping(self): + """Preserve the resource path, float32 conversion and crop.""" + resource = resources.files("giant_python").joinpath( + "assets", "psfs", "dil-17.tif" + ) + with resources.as_file(resource) as path: + original = tifffile.imread(str(path)).astype(np.float32) + with patch.object(inputs, "read_tiff", wraps=read_tiff) as reader: + template = inputs.default_psf(17) + self.assertEqual(Path(reader.call_args.args[0]).name, "dil-17.tif") + reader.assert_called_once() + self.assertEqual(template.dtype, np.dtype("float32")) + np.testing.assert_array_equal(template, original) + expected = original.copy() + expected[expected < expected.max() * np.exp(-3)] = 0 + rows = np.flatnonzero(np.any(expected != 0, axis=1)) + cols = np.flatnonzero(np.any(expected != 0, axis=0)) + row_slice = slice(rows[0], rows[-1] + 1) + col_slice = slice(cols[0], cols[-1] + 1) + expected = expected[row_slice, col_slice] + with patch.object( + inputs, "default_psf", wraps=inputs.default_psf + ) as loader: + result = inputs.load_psf(17, 2) + self.assertEqual(loader.call_args_list, [call(17), call(17)]) + self.assertEqual(set(result), {"DMD1", "DMD2"}) + for actual in result.values(): + self.assertEqual(actual.dtype, np.dtype("float32")) + np.testing.assert_array_equal(actual, expected) + self.assertFalse(np.shares_memory(result["DMD1"], result["DMD2"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_band_workflow.py b/tests/test_band_workflow.py new file mode 100644 index 0000000..9e8241b --- /dev/null +++ b/tests/test_band_workflow.py @@ -0,0 +1,460 @@ +"""Public band orchestration without vendor data or an expensive NMF fit. + +The real driver, per-path composition, low-res assembly, motion geometry, +high-res trace solver, result assembly and HDF5 writer run together. Only +external input reads, background/noise estimation and localization are mocked. +""" + +import unittest +from contextlib import ExitStack +from dataclasses import asdict +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch + +import numpy as np +import torch + +from giant_python.extraction.band import traces +from giant_python.extraction.band import workflow as wf +from giant_python.extraction.band.types import TrialTraceResult +from giant_python.io.experiment_summary import read_summary +from giant_python.models.params import ( + AnnotationOptions, + BandSiloParams, + ExecutionOptions, +) +from giant_python.models.trial_table import Slap2Info, TrialTable +from giant_python.pipeline.extract import extract_band_sources + + +class TestBandWorkflow(unittest.TestCase): + """Exercise the public API through the driver to saved results.""" + + def _run_mock_session(self, *, loaded=False, rois=False): + """Check schema, policy and motion values in a two-path session.""" + n_frames, side, n_sp = 120, 15, 49 + positions = [(r, c) for r in range(4, 11) for c in range(4, 11)] + smi = np.array( + [ + [r * side + c, index + 1] + for index, (r, c) in enumerate(positions) + ], + dtype=np.int32, + ) + frames = np.arange(n_frames, dtype=float) + 11 + psf = np.ones((3, 3), dtype=np.float32) + options = ExecutionOptions(max_workers=3, max_trials=2) + annotation = AnnotationOptions( + enabled=rois, interactive=False, operator="workflow test" + ) + science_args = dict( + analyze_hz=10, denoise_window_s=0.3, baseline_window_s=0.5 + ) + params = BandSiloParams(**science_args) + before = (asdict(params), asdict(options), asdict(annotation)) + + with TemporaryDirectory() as directory, ExitStack() as stack: + root = Path(directory) + # Deliberately not savedr/trial_table.h5, and not even on disk. + source = root / "selection" / "only-selected-trials.h5" + table = TrialTable( + source_path=source, + savedr=root / "results", + datadr=root / "raw", + filename=np.array( + [ + [ + f"p{d}-skip.dat", + f"p{d}-selected.dat", + f"p{d}-beyond-limit.dat", + ] + for d in range(2) + ], + dtype=object, + ), + slap2_info=Slap2Info( + first_line=np.full((2, 3), 11), + last_line=np.full((2, 3), 130), + ref_stack={"selection": "unchanged"}, + ), + motion_correction={ + "fn_adata": np.array( + [ + [ + f"p{d}-skip.h5", + f"p{d}-selected.h5", + f"p{d}-beyond-limit.h5", + ] + for d in range(2) + ], + dtype=object, + ) + }, + ) + filenames_before = table.filename.copy() + align_before = table.motion_correction["fn_adata"].copy() + lookup = { + "fastZ2RefZ": { + f"DMD{d + 1}": np.array([[10 + d]]) for d in range(2) + }, + "allSuperPixelIDs": { + f"DMD{d + 1}": np.arange(n_sp).reshape(-1, 1) + for d in range(2) + }, + "sparseMaskInds": {f"DMD{d + 1}": smi for d in range(2)}, + } + read_table = stack.enter_context( + patch.object(TrialTable, "from_h5", return_value=table) + ) + prepare = stack.enter_context( + patch.object(wf, "load_trial_table", wraps=wf.load_trial_table) + ) + stack.enter_context( + patch.object( + wf.inputs, "load_lookup_table", return_value=lookup + ) + ) + stack.enter_context( + patch.object( + wf.inputs, + "load_psf", + return_value={"DMD1": psf, "DMD2": psf}, + ) + ) + stack.enter_context( + patch.object( + wf.inputs, + "load_reference_stack", + return_value=(np.zeros((1, 1, side, side)), None, None), + ) + ) + stack.enter_context( + patch.object( + wf.geo, "build_subsample_matrix_inds", return_value=smi + ) + ) + keep = stack.enter_context( + patch.object( + wf, + "compute_keep_trials", + return_value=np.array( + [[False, True, True], [False, True, True]] + ), + ) + ) + stack.enter_context( + patch.object( + wf, + "read_align_info", + return_value=({"DMD1": 10.0, "DMD2": 10.0}, 2), + ) + ) + geometry = stack.enter_context( + patch.object( + wf, "build_user_roi_geometry", return_value=({}, {}) + ) + ) + roi_selection = { + "user_roi_masks": { + f"DMD{d + 1}": [np.ones((1, side, side), dtype=bool)] + for d in range(2) + }, + "user_roi_labels": { + f"DMD{d + 1}": [f"ROI {d}"] for d in range(2) + }, + "user_roi_superpixels": { + f"DMD{d + 1}": [np.array([0, 1])] for d in range(2) + }, + } + resolve_rois = stack.enter_context( + patch.object( + wf, "resolve_user_rois", return_value=roi_selection + ) + ) + + def read_trial( + trial_ix, + keep_trial, + dmd_ix, + samp_freq, + sp_ids, + datadr, + runtime_table, + num_channels, + all_channels=True, + ): + """Supply kept data and verify resolved read inputs.""" + self.assertIs(runtime_table["trial_table"], table) + self.assertEqual(runtime_table["source_path"], source) + self.assertEqual(num_channels, 2) + self.assertEqual(samp_freq, 10) + self.assertEqual(datadr, str(table.datadr)) + self.assertIn(trial_ix, (0, 1)) + if not keep_trial: + return None + self.assertEqual(trial_ix, 1) + self.assertEqual( + runtime_table["filename"][dmd_ix, trial_ix], + f"p{dmd_ix}-selected.dat", + ) + # Internal reference offsets, not saved displacement. + alignment = { + "DSframes": frames, + "motionDSr": np.full(n_frames, 2.0), + "motionDSc": np.full(n_frames, -1.0), + "motionDSz": np.zeros(n_frames), + "onlineYshift": np.full(n_frames, 4.0), + "onlineXshift": np.full(n_frames, 5.0), + "onlineZshift": np.full(n_frames, 6.0), + } + data = np.full( + (n_sp, n_frames), 20 + 10 * dmd_ix, dtype=np.float32 + ) + return { + "data": data, + "data_count": np.full_like(data, 2), + "data2": data * 2, + "data_count2": np.full_like(data, 2), + "alignment": alignment, + "ds_frames": frames, + } + + low_read = stack.enter_context( + patch.object( + wf.td, "read_band_trial_data", side_effect=read_trial + ) + ) + high_read = stack.enter_context( + patch.object( + traces, "read_band_trial_data", side_effect=read_trial + ) + ) + mapper = stack.enter_context( + patch.object( + wf, + "map_trials", + side_effect=lambda fn, items, max_workers, **kw: [ + fn(i) for i in items + ], + ) + ) + stack.enter_context( + patch.object( + wf, + "_estimate_background", + side_effect=lambda data, *args, **kw: data * 0.75, + ) + ) + stack.enter_context( + patch.object( + wf.bg, + "fit_noise_variance_model", + side_effect=lambda data, *args: (np.ones_like(data), 1, 0), + ) + ) + stack.enter_context( + patch.object( + wf, + "_localize", + return_value=( + np.ones((1, side, side), dtype=np.float32), + np.array([[0, 7, 7]]), + torch.ones((side * side, 1)), + torch.tensor( + [[0, 7, 7, 1, 1, 1]], dtype=torch.float32 + ), + np.array([2], dtype=np.float32), + ), + ) + ) + assemble = stack.enter_context( + patch.object( + wf, "assemble_path_summary", wraps=wf.assemble_path_summary + ) + ) + writer = stack.enter_context( + patch.object( + wf, + "write_summary", + wraps=wf.write_summary, + ) + ) + + input_value = table if loaded else source + summary = extract_band_sources( + input=input_value, + params=params, + execution=options, + annotations=annotation, + ) + self.assertIs(prepare.call_args.args[0], input_value) + prepare.assert_called_once() + if loaded: + read_table.assert_not_called() + else: + read_table.assert_called_once_with(source) + np.testing.assert_array_equal(table.filename, filenames_before) + np.testing.assert_array_equal( + table.motion_correction["fn_adata"], align_before + ) + self.assertEqual(table.source_path, source) + self.assertEqual( + keep.call_args.args[0][0, 1], + str(table.savedr / "motion_correction" / "p0-selected.h5"), + ) + self.assertEqual(low_read.call_count, 4) + self.assertEqual(high_read.call_count, 2) + self.assertEqual(mapper.call_count, 4) + for call in mapper.call_args_list: + self.assertEqual(call.args[2], options.max_workers) + self.assertEqual(list(call.args[1]), [0, 1]) + if rois: + geometry.assert_called_once() + self.assertFalse(resolve_rois.call_args.kwargs["interactive"]) + else: + geometry.assert_not_called() + resolve_rois.assert_not_called() + + writer.assert_called_once() + output = ( + table.savedr / "source_extraction" / "experiment_summary.h5" + ) + self.assertIs(writer.call_args.args[0], summary) + self.assertEqual(writer.call_args.args[1], output) + restored = read_summary(output) + self.assertEqual(len(summary.paths), 2) + self.assertEqual(len(restored.paths), 2) + self.assertEqual(summary.params["numChannels"], 2) + self.assertEqual(summary.params["operator"], "workflow test") + self.assertEqual(summary.params["draw_user_rois"], rois) + for dmd_ix, (path, saved) in enumerate( + zip(summary.paths, restored.paths) + ): + runtime_result = assemble.call_args_list[dmd_ix].args[0] + self.assertEqual(len(runtime_result.trace_results), 1) + self.assertIsInstance( + runtime_result.trace_results[0], TrialTraceResult + ) + self.assertEqual(path.path_index, dmd_ix) + np.testing.assert_array_equal(path.z_depths, [[10 + dmd_ix]]) + np.testing.assert_array_equal( + path.frame_info.trial_num_frames, [[n_frames]] + ) + np.testing.assert_array_equal( + path.frame_info.frame_line_idxs.ravel(), frames + ) + np.testing.assert_array_equal( + path.frame_info.offlineYshifts, -2 + ) + np.testing.assert_array_equal( + path.frame_info.offlineXshifts, 1 + ) + np.testing.assert_array_equal(path.frame_info.onlineYshifts, 4) + np.testing.assert_array_equal(path.frame_info.onlineXshifts, 5) + np.testing.assert_allclose( + path.global_f[0], n_sp * (10 + 5 * dmd_ix) + ) + np.testing.assert_array_equal(path.global_f, saved.global_f) + np.testing.assert_array_equal( + path.frame_info.offlineXshifts, + saved.frame_info.offlineXshifts, + ) + np.testing.assert_allclose( + path.sources[0].df_ls, + saved.sources[0].df_ls, + equal_nan=True, + ) + self.assertEqual(len(path.user_rois), int(rois)) + self.assertEqual(saved.annotation_enabled, rois) + if rois: + self.assertEqual(path.user_rois[0].label, f"ROI {dmd_ix}") + self.assertEqual(path.user_rois[0].f.shape, (2, n_frames)) + self.assertNotEqual( + summary.paths[0].global_f[0, 0], + summary.paths[1].global_f[0, 0], + ) + self.assertEqual( + before, (asdict(params), asdict(options), asdict(annotation)) + ) + self.assertIsNone(params.num_channels) + + def test_exact_nonstandard_selected_trial_table_path(self): + """Preserve the chosen filename through extraction and persistence.""" + self._run_mock_session() + + def test_loaded_table_is_not_reopened(self): + """Prepare a loaded model directly without a second file read.""" + self._run_mock_session(loaded=True) + + def test_split_policy_preserves_rois_and_caller_options(self): + """Split policy leaves scientific options unchanged.""" + self._run_mock_session(loaded=True, rois=True) + + def test_missing_channel_inference_fails_before_processing(self): + """Missing channel metadata raises before ROI resolution or writing.""" + params = BandSiloParams() + with TemporaryDirectory() as directory, ExitStack() as stack: + table = dict( + savedr=directory, + n_dmds=1, + fn_adata=[], + filename=[], + datadr=directory, + ) + stack.enter_context( + patch.object(wf, "load_trial_table", return_value=table) + ) + stack.enter_context(patch.object(wf.inputs, "load_lookup_table")) + stack.enter_context(patch.object(wf.inputs, "load_psf")) + stack.enter_context(patch.object(wf, "compute_keep_trials")) + stack.enter_context( + patch.object(wf, "read_align_info", return_value=({}, None)) + ) + rois = stack.enter_context( + patch.object(wf, "_resolve_session_user_rois") + ) + process = stack.enter_context(patch.object(wf, "_process_dmd")) + writer = stack.enter_context(patch.object(wf, "write_summary")) + with self.assertRaisesRegex( + ValueError, "num_channels.*determined" + ): + wf.extract_band_sources("selected.h5", params) + rois.assert_not_called() + process.assert_not_called() + writer.assert_not_called() + self.assertIsNone(params.num_channels) + + def test_enabled_empty_annotations_keep_resolution_context(self): + """An enabled empty selection retains geometry and headless policy.""" + table, lookup = {"n_dmds": 2}, {"lookup": "unchanged"} + geometry, references, selection = {}, {"DMD1": "reference.tif"}, {} + with ( + patch.object( + wf, + "build_user_roi_geometry", + return_value=(geometry, references), + ) as build, + patch.object( + wf, "resolve_user_rois", return_value=selection + ) as resolve, + ): + result = wf._resolve_session_user_rois( + "results", + table, + lookup, + AnnotationOptions(enabled=True, interactive=False), + ) + build.assert_called_once_with(table, lookup) + resolve.assert_called_once_with( + Path("results") / "annotations", + 2, + geometry, + interactive=False, + ref_files=references, + ) + self.assertIs(result, selection) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_bandsilo_annotate.py b/tests/test_bandsilo_annotate.py index 02bab86..fd4543b 100644 --- a/tests/test_bandsilo_annotate.py +++ b/tests/test_bandsilo_annotate.py @@ -1,12 +1,21 @@ -"""Tests for giant_python.bandsilo.annotate (pure seams + load/fail-fast).""" +"""Tests for band annotation's pure seams and split-policy orchestration.""" +import os import tempfile import unittest +from copy import deepcopy +from unittest import mock import numpy as np -from giant_python.bandsilo import annotate as an -from giant_python.bandsilo.gui import save_annotations_h5 +from giant_python.extraction.band import annotation as an +from giant_python.io.annotations import save_annotations_h5 +from giant_python.models import ( + AnnotationOptions, + BandSiloParams, + ExecutionOptions, + TrialTable, +) class TestFirstValidTrial(unittest.TestCase): @@ -147,5 +156,72 @@ def test_missing_interactive_draws(self): self.assertIs(out, sentinel) +class TestAnnotationService(unittest.TestCase): + """Annotation validates scoped options before IO without legacy bridges.""" + + def test_split_options_and_in_memory_input(self): + """The explicit step uses GUI/run policy and never mutates callers.""" + input = TrialTable() + params = BandSiloParams() + execution = ExecutionOptions(verbose=True) + annotations = AnnotationOptions(interactive=False, operator="Ada") + before = deepcopy((params, execution, annotations)) + table = { + "fn_adata": np.array([["a.h5"]]), + "filename": np.array([["trial"]]), + "datadr": "/data", + "annotation_save_dr": "/annotations", + "moco_save_dr": "/motion", + "n_dmds": 1, + } + with ( + mock.patch.object( + an, "load_trial_table", return_value=table + ) as load, + mock.patch.object( + an, "compute_keep_trials", return_value=np.array([[True]]) + ), + mock.patch.object(an.inputs, "load_lookup_table"), + mock.patch.object( + an, "build_user_roi_geometry", return_value=({}, {}) + ), + mock.patch.object(an, "resolve_user_rois") as resolve, + mock.patch.object(an, "log") as log, + ): + result = an.annotate_band_rois( + input, + params=params, + execution=execution, + annotations=annotations, + ) + load.assert_called_once_with(input) + resolve.assert_called_once_with( + "/annotations", 1, {}, interactive=False, ref_files={} + ) + self.assertEqual( + result, os.path.join("/annotations", "annotations.h5") + ) + self.assertTrue(all(call.args[1] for call in log.call_args_list)) + self.assertEqual((params, execution, annotations), before) + self.assertFalse(annotations.enabled) + + def test_mixed_options_rejected_before_io(self): + """Mis-scoped dictionaries and removed keywords cannot load data.""" + with mock.patch.object(an, "load_trial_table") as load: + for kwargs in ( + {"params": {"max_workers": 2}}, + {"params": {"operator": "Ada"}}, + {"params": {"scan_mode": "band"}}, + {"execution": {"enabled": True}}, + {"annotations": {"draw_user_rois": True}}, + {"params_in": {}}, + ): + with self.subTest(kwargs=kwargs): + with self.assertRaises(TypeError): + an.annotate_band_rois("trial.h5", **kwargs) + load.assert_not_called() + self.assertFalse(hasattr(an, "_resolve_params")) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_bandsilo_background.py b/tests/test_bandsilo_background.py index 41afb23..3bfd0c4 100644 --- a/tests/test_bandsilo_background.py +++ b/tests/test_bandsilo_background.py @@ -1,11 +1,11 @@ -"""Tests for giant_python.bandsilo.background (Phase 4 kernels). +"""Tests for canonical band background, activity, and motion kernels. Covers the motion-binning, PSF/convolution, interpolated-background, noise-model, and rho kernels with synthetic fixtures. ``build_interp_data`` is additionally cross-checked against a verbatim copy of the reference in the project's development notes; here we assert its structural behavior. The rho path is exercised end-to-end on a small synthetic geometry built from the real -:mod:`giant_python.bandsilo.geometry` helpers, and checked for linearity in the +:mod:`giant_python.extraction.band.geometry` helpers, checked for linearity in residual (rho is a linear projection of the residual). """ @@ -14,8 +14,10 @@ import numpy as np import torch -from giant_python.bandsilo import background as bg -from giant_python.bandsilo import geometry as geo +from giant_python.extraction.band import activity +from giant_python.extraction.band import background as bg +from giant_python.extraction.band import geometry as geo +from giant_python.extraction.band import motion_binning, operators def _small_geometry(seed=0, positions_per_plane=None): @@ -53,31 +55,33 @@ def _small_geometry(seed=0, positions_per_plane=None): ref_d, ref_c, ref_r = geo.ref_pixs_to_drc( subsample_matrix_inds[:, 0], npc, npr ) - sparse_h_inds, sparse_h_vals = geo.build_sparse_h( + sparse_h_inds, sparse_h_vals = operators.build_sparse_h( subsample_matrix_inds, psf2d, npc, npr ) umyx = np.array([[0, 0], [1, 0]], dtype=np.float64) - sel_pix_mask, sel_pix_idxs = bg.build_selected_pixel_mask( + sel_pix_mask, sel_pix_idxs = geo.build_selected_pixel_mask( umyx, ref_d, ref_r, ref_c, num_fast_zs, npc, npr, psf2d ) - psf_tensor, psf_center, psf_tensor_exp, psf_center_exp = bg.expand_psf( - psf2d + psf_tensor, psf_center, psf_tensor_exp, psf_center_exp = ( + operators.expand_psf(psf2d) ) d_mats = [] d_mats_exp = [] for z in range(num_fast_zs): - _, sel_2d = bg.selected_pixels_2d_for_plane(sel_pix_idxs, z, npc, npr) + _, sel_2d = geo.selected_pixels_2d_for_plane(sel_pix_idxs, z, npc, npr) d_mats.append( - bg.build_convolution_matrix(sel_2d, psf_tensor, psf_center) + operators.build_convolution_matrix(sel_2d, psf_tensor, psf_center) ) d_mats_exp.append( - bg.build_convolution_matrix(sel_2d, psf_tensor_exp, psf_center_exp) + operators.build_convolution_matrix( + sel_2d, psf_tensor_exp, psf_center_exp + ) ) - h_mots = bg.build_motion_h_matrices( + h_mots = operators.build_motion_h_matrices( sparse_h_inds, sparse_h_vals, umyx, sel_pix_idxs, num_super_pixels, npr ) @@ -107,7 +111,7 @@ def test_bin_motion_groups_identical_rounded_vectors(self): mr = np.array([0.1, 0.2, 5.0]) mc = np.array([0.0, 0.0, 0.0]) mz = np.array([0.0, 0.0, 0.0]) - unique_motion, mot_inds = bg.bin_motion(mr, mc, mz) + unique_motion, mot_inds = motion_binning.bin_motion(mr, mc, mz) self.assertEqual(unique_motion.shape[1], 3) self.assertEqual(mot_inds[0], mot_inds[1]) self.assertNotEqual(mot_inds[0], mot_inds[2]) @@ -121,7 +125,7 @@ def test_select_motion_bins_filters_z_and_count(self): ) mot_inds = np.array([0] * 5 + [1] * 2 + [2] * 5) motion_z = np.zeros(12) - keep, frames = bg.select_motion_bins( + keep, frames = motion_binning.select_motion_bins( unique_motion, mot_inds, motion_z, z_thresh=1.5, min_frames=3 ) self.assertEqual(list(keep), [0]) @@ -132,7 +136,9 @@ def test_bin_motion_yx_marks_dropped_frames(self): mr = np.array([0.0, 0.0, 1.0, 1.0]) mc = np.array([0.0, 0.0, 0.0, 0.0]) frames_to_keep = np.array([True, True, True, False]) - umyx, mot_inds_yx = bg.bin_motion_yx(mr, mc, frames_to_keep) + umyx, mot_inds_yx = motion_binning.bin_motion_yx( + mr, mc, frames_to_keep + ) self.assertEqual(mot_inds_yx[3], -1) self.assertEqual(mot_inds_yx[0], mot_inds_yx[1]) self.assertEqual(umyx.shape[1], 2) @@ -148,11 +154,11 @@ def test_mask_and_coords_round_trip(self): ref_c = np.array([5]) umyx = np.array([[0, 0]], dtype=float) psf2d = np.ones((3, 3), dtype=np.float32) - mask, idxs = bg.build_selected_pixel_mask( + mask, idxs = geo.build_selected_pixel_mask( umyx, ref_d, ref_r, ref_c, 1, 12, 12, psf2d ) self.assertEqual(int(mask.sum()), idxs.size) - coords = bg.pixel_coords_from_idxs(idxs, 12, 12) + coords = geo.pixel_coords_from_idxs(idxs, 12, 12) # every decoded coordinate should be marked in the mask for z, r, c in coords: self.assertTrue(mask[z, r, c]) @@ -160,9 +166,9 @@ def test_mask_and_coords_round_trip(self): def test_selected_pixels_2d_for_plane(self): """Plane selection returns the in-plane row/col for that z only.""" idxs = np.array([0 * 25 + 2 * 5 + 3, 1 * 25 + 4 * 5 + 1]) - z0, sel0 = bg.selected_pixels_2d_for_plane(idxs, 0, 5, 5) + z0, sel0 = geo.selected_pixels_2d_for_plane(idxs, 0, 5, 5) self.assertEqual(sel0.tolist(), [[2, 3]]) - z1, sel1 = bg.selected_pixels_2d_for_plane(idxs, 1, 5, 5) + z1, sel1 = geo.selected_pixels_2d_for_plane(idxs, 1, 5, 5) self.assertEqual(sel1.tolist(), [[4, 1]]) @@ -173,7 +179,7 @@ def test_expand_psf_normalized_and_larger(self): """Both PSFs sum to 1 and the expanded one is ex_fac larger.""" yy, xx = np.mgrid[-2:3, -2:3] psf2d = np.exp(-(yy**2 + xx**2) / 2.0).astype(np.float32) - pt, pc, pte, pce = bg.expand_psf(psf2d, ex_fac=2) + pt, pc, pte, pce = operators.expand_psf(psf2d, ex_fac=2) self.assertAlmostEqual(float(pt.sum()), 1.0, places=5) self.assertAlmostEqual(float(pte.sum()), 1.0, places=5) self.assertEqual(tuple(pte.shape), (10, 10)) @@ -184,9 +190,9 @@ def test_convolution_matrix_values(self): """D[t, s] equals the PSF weight at the target-source offset.""" yy, xx = np.mgrid[-2:3, -2:3] psf2d = np.exp(-(yy**2 + xx**2) / 2.0).astype(np.float32) - pt, pc, _, _ = bg.expand_psf(psf2d) + pt, pc, _, _ = operators.expand_psf(psf2d) sel_2d = np.array([[5, 5], [6, 5], [5, 8]]) - d = bg.build_convolution_matrix(sel_2d, pt, pc) + d = operators.build_convolution_matrix(sel_2d, pt, pc) # self entry == center weight self.assertAlmostEqual(float(d[0, 0]), float(pt[pc[0], pc[1]]), 5) # offset (1, 0): target row+1 -> psf[center+1, center] @@ -199,7 +205,7 @@ def test_convolution_matrix_values(self): def test_convolution_matrix_empty(self): """An empty plane yields a 0x0 matrix.""" pt = torch.ones((3, 3)) - d = bg.build_convolution_matrix( + d = operators.build_convolution_matrix( np.zeros((0, 2), dtype=int), pt, (1, 1) ) self.assertEqual(tuple(d.shape), (0, 0)) @@ -340,6 +346,7 @@ def test_no_nan_matches_brute_median(self): """With no NaNs, the baseline is a centered moving median.""" def brute_rolling_median(data, window): + """Compute clipped centered medians from finite row samples.""" n = data.shape[1] out = np.full(data.shape, np.nan, dtype=float) for r in range(data.shape[0]): @@ -472,11 +479,11 @@ def test_compute_rho_shape_and_linearity(self): dmd_pixels_per_row=g["npr"], psf2d=g["psf2d"], ) - rho = bg.compute_rho(residual, mot_yx, **kwargs) + rho = activity.compute_rho(residual, mot_yx, **kwargs) self.assertEqual(rho.shape, (g["sel_pix_idxs"].shape[0], n_frames)) self.assertTrue(np.any(np.isfinite(rho))) - rho2 = bg.compute_rho(2.0 * residual, mot_yx, **kwargs) + rho2 = activity.compute_rho(2.0 * residual, mot_yx, **kwargs) finite = np.isfinite(rho) & np.isfinite(rho2) np.testing.assert_allclose( rho2[finite], 2.0 * rho[finite], rtol=1e-4, atol=1e-5 @@ -494,7 +501,7 @@ def test_compute_rho_handles_empty_motion_and_empty_plane(self): # All frames in bin 0; bin 1 gets none -> motion_frames empty branch mot_yx = np.zeros(n_frames, dtype=np.int32) # Add a third, always-empty z-plane -> new_ncols == 0 branch - rho = bg.compute_rho( + rho = activity.compute_rho( residual, mot_yx, unique_motion_to_keep_yx=g["umyx"], @@ -526,7 +533,7 @@ def test_compute_rho_skips_plane_with_no_valid_columns(self): .astype(np.float32) ) mot_yx = np.array([0, 1] * (n_frames // 2), dtype=np.int32) - rho = bg.compute_rho( + rho = activity.compute_rho( residual, mot_yx, unique_motion_to_keep_yx=g["umyx"], @@ -554,7 +561,7 @@ def test_mask_high_nan_rho(self): [[1.0, 2.0, 3.0, 4.0], [np.nan, np.nan, np.nan, 1.0]], dtype=np.float32, ) - nan_ct = bg.mask_high_nan_rho(rho, thresh=0.5) + nan_ct = activity.mask_high_nan_rho(rho, thresh=0.5) self.assertAlmostEqual(nan_ct[0], 0.0) self.assertAlmostEqual(nan_ct[1], 0.75) self.assertTrue(np.all(np.isnan(rho[1]))) @@ -562,7 +569,7 @@ def test_mask_high_nan_rho(self): def test_decay_kernel_normalized(self): """The decay kernel sums to 1 and rises to its last sample.""" - k = bg.decay_kernel_1d(0.15, 80.0) + k = activity.decay_kernel_1d(0.15, 80.0) self.assertAlmostEqual(float(k.sum()), 1.0, places=6) self.assertEqual(np.argmax(k), len(k) - 1) @@ -574,8 +581,8 @@ def test_smooth_rho_preserves_constant(self): np.full((1, 50), np.nan, dtype=np.float32), ] ) - k = bg.decay_kernel_1d(0.05, 80.0) - out = bg.smooth_rho(rho, k) + k = activity.decay_kernel_1d(0.05, 80.0) + out = activity.smooth_rho(rho, k) # interior samples of the constant row remain ~2.0 np.testing.assert_allclose(out[0, 10:40], 2.0, rtol=1e-4) self.assertTrue(np.all(np.isnan(out[1]))) @@ -583,8 +590,8 @@ def test_smooth_rho_preserves_constant(self): def test_smooth_rho_all_nan_chunk(self): """A chunk with no finite data is passed through unchanged.""" rho = np.full((3, 20), np.nan, dtype=np.float32) - k = bg.decay_kernel_1d(0.05, 80.0) - out = bg.smooth_rho(rho, k) + k = activity.decay_kernel_1d(0.05, 80.0) + out = activity.smooth_rho(rho, k) self.assertTrue(np.all(np.isnan(out))) diff --git a/tests/test_bandsilo_baseline.py b/tests/test_bandsilo_baseline.py index b6f2024..27fcf83 100644 --- a/tests/test_bandsilo_baseline.py +++ b/tests/test_bandsilo_baseline.py @@ -1,4 +1,4 @@ -"""Tests for giant_python.bandsilo.baseline (Phase 7 F0 / dF-F). +"""Tests for giant_python.numerics.baseline (Phase 7 F0 / dF-F). Covers the NaN-aware moving mean, the rolling-min-envelope ``compute_f0`` baseline estimator, and the dF/F assembly. ``compute_f0`` is additionally @@ -10,7 +10,7 @@ import numpy as np -from giant_python.bandsilo import baseline as bl +from giant_python.numerics import baseline as bl class TestMovmeanNan(unittest.TestCase): diff --git a/tests/test_bandsilo_geometry.py b/tests/test_bandsilo_geometry.py index 1217b9b..c7587dd 100644 --- a/tests/test_bandsilo_geometry.py +++ b/tests/test_bandsilo_geometry.py @@ -1,4 +1,4 @@ -"""Tests for giant_python.bandsilo.geometry.""" +"""Tests for canonical band geometry, input loading, and sparse operators.""" import os import tempfile @@ -8,19 +8,21 @@ import numpy as np import tifffile -from giant_python.bandsilo.geometry import ( +from giant_python.extraction.band.geometry import ( _pad_to, build_combined_psf, - build_sparse_h, build_subsample_matrix_inds, + ref_pixs_to_drc, + threshold_and_crop_psf, +) +from giant_python.extraction.band.inputs import ( default_psf, find_reference_file, load_lookup_table, load_psf, load_reference_stack, - ref_pixs_to_drc, - threshold_and_crop_psf, ) +from giant_python.extraction.band.operators import build_sparse_h class TestRefPixsToDrc(unittest.TestCase): @@ -35,10 +37,10 @@ def test_known_mapping(self): class TestSubsampleMatrixInds(unittest.TestCase): - """build_subsample_matrix_inds picks median open pixels.""" + """Preserve the original middle-by-input-order representative pixel.""" def test_median_reference_pixel(self): - """Odd-length pixels use the median; even-length use the middle.""" + """Sorted pixels select their middle entry for odd and even lengths.""" all_ids = np.array([[1], [2]], dtype=np.int32) mask = np.array( [[10, 1], [12, 1], [14, 1], [20, 2], [22, 2]], dtype=np.int32 @@ -48,13 +50,14 @@ def test_median_reference_pixel(self): out, np.array([[11, 1], [21, 2]], dtype=np.int32) ) - def test_odd_length_uses_median_not_middle_index(self): - """Unsorted odd-length open pixels pick median, not middle by order.""" + def test_odd_length_preserves_middle_index(self): + """The structural refactor does not change unsorted pixel selection.""" all_ids = np.array([[1]], dtype=np.int32) - # 0-based open pixels: [13, 9, 11] -> median 11, middle index -> 9 + # The original implementation selects 9, not the sorted median 11. + # Changing that rule would be a separate scientific behavior change. mask = np.array([[14, 1], [10, 1], [12, 1]], dtype=np.int32) out = build_subsample_matrix_inds(all_ids, mask) - np.testing.assert_array_equal(out, np.array([[11, 1]], dtype=np.int32)) + np.testing.assert_array_equal(out, np.array([[9, 1]], dtype=np.int32)) class TestSparseH(unittest.TestCase): diff --git a/tests/test_bandsilo_gui.py b/tests/test_bandsilo_gui.py index b2f5fd4..1ab61e8 100644 --- a/tests/test_bandsilo_gui.py +++ b/tests/test_bandsilo_gui.py @@ -1,4 +1,4 @@ -"""Tests for giant_python.bandsilo.gui (pure + annotation-IO paths).""" +"""Tests for canonical band ROI geometry and annotation IO.""" import os import tempfile @@ -7,14 +7,14 @@ import h5py import numpy as np -from giant_python.bandsilo.gui import ( - _h5_scalar_str, +from giant_python.extraction.band.annotation import load_annotations_h5 +from giant_python.extraction.band.rois import ( _read_roi_records, compute_user_roi_geometry, - load_annotations_h5, - save_annotations_h5, user_roi_superpixel_lists_from_masks, ) +from giant_python.io.annotations import _h5_scalar_str, save_annotations_h5 +from giant_python.io.hdf5 import _read_group def _str_dt(): @@ -219,7 +219,7 @@ def test_defaults_and_missing_mask(self): grp.create_group("roi_000") with h5py.File(path, "r") as f: masks, labels, recs, bad = _read_roi_records( - f["Path1"], expected + _read_group(f["Path1"], row_major=True), expected ) self.assertFalse(bad) self.assertEqual(labels, ["ROI1"]) diff --git a/tests/test_bandsilo_hdf5.py b/tests/test_bandsilo_hdf5.py index 286f075..9c1acad 100644 --- a/tests/test_bandsilo_hdf5.py +++ b/tests/test_bandsilo_hdf5.py @@ -1,4 +1,4 @@ -"""Tests for giant_python.bandsilo.hdf5.""" +"""Tests for shared HDF5 IO and band runtime input preparation.""" import os import tempfile @@ -7,19 +7,21 @@ import h5py import numpy as np -from giant_python.bandsilo.hdf5 import ( - _decode_h5_strings, - _decode_one, - _reshape_1d, +from giant_python.extraction.band.inputs import ( _resolve_fn_adata, compute_keep_trials, load_alignment_data_h5, - load_struct_from_h5, load_trial_table, read_align_info, +) +from giant_python.io.hdf5 import ( + _decode_one, + _decode_strings, + load_struct_h5, to_serializable, write_dict_to_h5group, ) +from giant_python.io.slap2 import _reshape_1d def _str_dt(): @@ -47,7 +49,7 @@ def test_passthrough(self): class TestDecodeHelpers(unittest.TestCase): - """_decode_one / _decode_h5_strings handle bytes, str, arrays.""" + """_decode_one / _decode_strings handle bytes, str, arrays.""" def test_decode_one(self): """bytes decode to str; str stays str.""" @@ -56,18 +58,18 @@ def test_decode_one(self): def test_decode_scalar_and_array(self): """Scalar bytes -> str; object array -> decoded object array.""" - self.assertEqual(_decode_h5_strings(b"z"), "z") - arr = _decode_h5_strings(np.array([b"a", b"b"], dtype=object)) + self.assertEqual(_decode_strings(b"z"), "z") + arr = _decode_strings(np.array([b"a", b"b"], dtype=object)) self.assertEqual(list(arr), ["a", "b"]) def test_decode_empty_array(self): """An empty array is returned unchanged (no vectorize crash).""" - out = _decode_h5_strings(np.array([], dtype=object)) + out = _decode_strings(np.array([], dtype=object)) self.assertEqual(out.size, 0) class TestReadDataset(unittest.TestCase): - """load_struct_from_h5 read-side orientation and string handling.""" + """load_struct_h5 read-side orientation and string handling.""" def test_scalar_string(self): """A scalar string dataset reads back as plain str.""" @@ -75,7 +77,7 @@ def test_scalar_string(self): path = os.path.join(d, "s.h5") with h5py.File(path, "w") as f: f.create_dataset("datadr", data="hello", dtype=_str_dt()) - out = load_struct_from_h5(path) + out = load_struct_h5(path) self.assertEqual(out["datadr"], "hello") def test_single_element_string_array(self): @@ -88,7 +90,7 @@ def test_single_element_string_array(self): data=np.array(["x"], dtype=object), dtype=_str_dt(), ) - out = load_struct_from_h5(path) + out = load_struct_h5(path) self.assertEqual(out["name"], "x") def test_string_grid_column_major_transposed(self): @@ -98,7 +100,7 @@ def test_string_grid_column_major_transposed(self): grid = np.array([["a", "b", "c"], ["d", "e", "f"]], dtype=object) with h5py.File(path, "w") as f: f.create_dataset("filename", data=grid, dtype=_str_dt()) - out = load_struct_from_h5(path) + out = load_struct_h5(path) self.assertEqual(out["filename"].shape, (3, 2)) def test_numeric_transpose_and_row_major(self): @@ -108,13 +110,13 @@ def test_numeric_transpose_and_row_major(self): col = os.path.join(d, "c.h5") with h5py.File(col, "w") as f: f.create_dataset("m", data=arr) - self.assertEqual(load_struct_from_h5(col)["m"].shape, (3, 2)) + self.assertEqual(load_struct_h5(col)["m"].shape, (3, 2)) rowm = os.path.join(d, "r.h5") with h5py.File(rowm, "w") as f: f.create_dataset("row_major", data=1) f.create_dataset("m", data=arr) - got = load_struct_from_h5(rowm)["m"] + got = load_struct_h5(rowm)["m"] self.assertEqual(got.shape, (2, 3)) np.testing.assert_array_equal(got, arr) @@ -126,7 +128,7 @@ def test_nested_group(self): f.create_dataset("row_major", data=1) g = f.create_group("inner") g.create_dataset("v", data=np.arange(3)) - out = load_struct_from_h5(path) + out = load_struct_h5(path) np.testing.assert_array_equal(out["inner"]["v"], np.arange(3)) @@ -152,7 +154,7 @@ def test_round_trip(self): path = os.path.join(d, "w.h5") with h5py.File(path, "w") as f: write_dict_to_h5group(f, payload) - out = load_struct_from_h5(path) + out = load_struct_h5(path) self.assertNotIn("empty", out) self.assertNotIn("missing", out) diff --git a/tests/test_bandsilo_nmf.py b/tests/test_bandsilo_nmf.py index 1f13892..72fe30e 100644 --- a/tests/test_bandsilo_nmf.py +++ b/tests/test_bandsilo_nmf.py @@ -1,4 +1,4 @@ -"""Tests for giant_python.bandsilo.nmf (Phase 6 source localization). +"""Tests for canonical band localization (Phase 6 source localization). Covers the profile/init kernels, the least-squares/SNR helpers, and the full ``fit_sources`` driver (multiplicative NMF + Adam Gaussian fit + variance @@ -13,8 +13,8 @@ import numpy as np import torch -from giant_python.bandsilo import background as bg -from giant_python.bandsilo import nmf +from giant_python.extraction.band import geometry +from giant_python.extraction.band import localization as nmf from tests.test_bandsilo_background import _small_geometry @@ -136,7 +136,7 @@ def _setup(self, seed=0): n_pix = nz * npc * npr sel = g["sel_pix_idxs"] pct = torch.tensor( - bg.pixel_coords_from_idxs(sel, npc, npr), dtype=torch.float32 + geometry.pixel_coords_from_idxs(sel, npc, npr), dtype=torch.float32 ) seeds = np.array([[0.0, 7.0, 7.0], [1.0, 7.0, 8.0]]) params = nmf.init_source_params(seeds) @@ -201,7 +201,7 @@ def _inputs(self, seed=0): npc, npr, nz = g["npc"], g["npr"], g["num_fast_zs"] n_pix = nz * npc * npr sel = g["sel_pix_idxs"] - coords = bg.pixel_coords_from_idxs(sel, npc, npr) + coords = geometry.pixel_coords_from_idxs(sel, npc, npr) seeds = np.array([[0.0, 7.0, 7.0], [0.0, 7.0, 9.0], [1.0, 7.0, 8.0]]) rng = np.random.default_rng(seed + 100) n_frames = 16 diff --git a/tests/test_bandsilo_peaks.py b/tests/test_bandsilo_peaks.py index 6a46e8d..8a40524 100644 --- a/tests/test_bandsilo_peaks.py +++ b/tests/test_bandsilo_peaks.py @@ -1,4 +1,4 @@ -"""Tests for giant_python.bandsilo.peaks (Phase 5 peak detection). +"""Tests for giant_python.numerics.peaks (Phase 5 peak detection). Covers the integrated-Gaussian forward/Jacobian kernels, the bounded LM solver, the per-plane detector, and the 3-D ``get_act_im_peaks`` driver with @@ -12,7 +12,7 @@ import numpy as np -from giant_python.bandsilo import peaks as pk +from giant_python.numerics import peaks as pk def _grid(height, width): @@ -39,29 +39,26 @@ def test_anisotropic_total_integral(self): total = pk.gaussian_peaks_integrated(theta, _grid(101, 27)).sum() self.assertAlmostEqual(total, 3.0 * 2 * np.pi * 8.0 * 0.7, places=6) - def test_equal_widths_match_legacy_isotropic(self): + def test_four_column_parameters_rejected(self): + """A missing x width fails instead of silently sharing the y width.""" yx = _grid(25, 25) - isotropic = np.array([[2.0, 12.3, 10.7, 0.9]]) - anisotropic = np.column_stack([isotropic, isotropic[:, 3]]) - np.testing.assert_allclose( - pk.gaussian_peaks_integrated(anisotropic, yx), - pk.gaussian_peaks_integrated(isotropic, yx), - rtol=1e-12, - ) + theta = np.array([[2.0, 12.3, 10.7, 0.9]]) + with self.assertRaisesRegex(ValueError, r"shape \(N, 5\).*sigma_x"): + pk.gaussian_peaks_integrated(theta, yx) def test_total_integral(self): """Integrated Gaussian summed over a wide grid gives amp*2*pi*s^2.""" yx = _grid(60, 60) amp, s = 3.0, 2.0 - theta = np.array([[amp, 30.0, 30.0, s]]) + theta = np.array([[amp, 30.0, 30.0, s, s]]) total = pk.gaussian_peaks_integrated(theta, yx).sum() self.assertAlmostEqual(total, amp * 2 * np.pi * s**2, places=3) def test_superposition(self): """Two Gaussians sum to the sum of their individual renders.""" yx = _grid(30, 30) - t1 = np.array([[2.0, 10.0, 12.0, 1.5]]) - t2 = np.array([[3.0, 20.0, 18.0, 1.0]]) + t1 = np.array([[2.0, 10.0, 12.0, 1.5, 1.5]]) + t2 = np.array([[3.0, 20.0, 18.0, 1.0, 1.0]]) both = pk.gaussian_peaks_integrated(np.vstack([t1, t2]), yx) sep = pk.gaussian_peaks_integrated( t1, yx @@ -69,16 +66,67 @@ def test_superposition(self): np.testing.assert_allclose(both, sep, rtol=1e-10) +class TestParameterContract(unittest.TestCase): + """Gaussian kernels and both fit stages require five explicit columns.""" + + def test_invalid_parameter_shapes_rejected(self): + """Reject missing widths, extra columns and non-matrix inputs early.""" + yx = _grid(3, 3) + for shape in [(1, 4), (0, 4), (1, 6), (1, 0), (5,), (1, 1, 5)]: + theta = np.ones(shape) + for kernel in [ + pk.gaussian_peaks_integrated, + pk._gaussian_peaks_integrated_val_jac, + pk._lsq_curvefit, + pk._fit_isotropic_then_anisotropic, + ]: + with self.subTest(shape=shape, kernel=kernel.__name__): + args = (theta, yx) + if kernel in [ + pk._lsq_curvefit, + pk._fit_isotropic_then_anisotropic, + ]: + args += ( + np.zeros(9), + np.zeros(theta.size), + np.ones(theta.size), + ) + with self.assertRaisesRegex( + ValueError, r"shape \(N, 5\).*sigma_y, sigma_x" + ): + kernel(*args) + + def test_bounds_cannot_broadcast(self): + """Fitters reject scalar, four-column and unflattened bounds.""" + theta = np.array([[1.0, 1.0, 1.0, 0.5, 0.5]]) + for fitter in [pk._lsq_curvefit, pk._fit_isotropic_then_anisotropic]: + for shape in [(), (1,), (4,), (1, 5)]: + for invalid_lower in [True, False]: + lb = np.zeros(shape if invalid_lower else (5,)) + ub = np.ones((5,) if invalid_lower else shape) + with self.subTest( + fitter=fitter.__name__, + shape=shape, + invalid_lower=invalid_lower, + ): + with self.assertRaisesRegex( + ValueError, r"bounds must have shape \(5\*N,\)" + ): + fitter(theta, _grid(3, 3), np.zeros(9), lb, ub) + + class TestValJac(unittest.TestCase): """_gaussian_peaks_integrated_val_jac value + analytical Jacobian.""" def test_anisotropic_jacobian_matches_finite_difference(self): """Both width derivatives are independent and correctly ordered.""" yx = _grid(35, 19) - theta = np.array([ - [2.0, 16.3, 7.7, 8.0, 0.7], - [1.2, 21.0, 12.0, 3.0, 1.0], - ]) + theta = np.array( + [ + [2.0, 16.3, 7.7, 8.0, 0.7], + [1.2, 21.0, 12.0, 3.0, 1.0], + ] + ) val, jac = pk._gaussian_peaks_integrated_val_jac(theta, yx) np.testing.assert_allclose( val, pk.gaussian_peaks_integrated(theta, yx), rtol=1e-12 @@ -99,7 +147,9 @@ def test_anisotropic_jacobian_matches_finite_difference(self): def test_value_matches_forward(self): """The val_jac value equals the forward-only evaluation.""" yx = _grid(20, 20) - theta = np.array([[2.0, 9.3, 11.7, 1.4], [1.5, 14.0, 6.0, 1.1]]) + theta = np.array( + [[2.0, 9.3, 11.7, 1.4, 1.4], [1.5, 14.0, 6.0, 1.1, 1.1]] + ) val, _ = pk._gaussian_peaks_integrated_val_jac(theta, yx) fwd = pk.gaussian_peaks_integrated(theta, yx) np.testing.assert_allclose(val, fwd, rtol=1e-12) @@ -107,7 +157,9 @@ def test_value_matches_forward(self): def test_jacobian_matches_finite_difference(self): """Analytical Jacobian matches a central finite-difference estimate.""" yx = _grid(16, 16) - theta = np.array([[2.0, 8.3, 7.7, 1.3], [1.2, 11.0, 5.0, 0.9]]) + theta = np.array( + [[2.0, 8.3, 7.7, 1.3, 1.3], [1.2, 11.0, 5.0, 0.9, 0.9]] + ) _, jac = pk._gaussian_peaks_integrated_val_jac(theta, yx) flat = theta.ravel().copy() eps = 1e-6 @@ -116,8 +168,8 @@ def test_jacobian_matches_finite_difference(self): hi[k] += eps lo = flat.copy() lo[k] -= eps - v_hi = pk.gaussian_peaks_integrated(hi.reshape(-1, 4), yx) - v_lo = pk.gaussian_peaks_integrated(lo.reshape(-1, 4), yx) + v_hi = pk.gaussian_peaks_integrated(hi.reshape(-1, 5), yx) + v_lo = pk.gaussian_peaks_integrated(lo.reshape(-1, 5), yx) fd = (v_hi - v_lo) / (2 * eps) np.testing.assert_allclose(jac[:, k], fd, rtol=1e-4, atol=1e-4) @@ -152,7 +204,7 @@ def test_max_nfev_early_break(self): """max_nfev=1 returns the (clipped) initial parameters immediately.""" yx = _grid(11, 11) ydata = pk.gaussian_peaks_integrated( - np.array([[4.0, 5.0, 5.0, 1.0]]), yx + np.array([[4.0, 5.0, 5.0, 1.0, 1.0]]), yx ) theta0 = np.array([[3.0, 5.0, 5.0, 1.0, 1.0]]) lb, ub = pk._make_bounds(theta0[:, 1:3], 11, 11) @@ -168,11 +220,11 @@ def test_non_improving_step_increases_damping(self): yx = _grid(21, 21) # data has its peak far from the seed; bounds pin the mean near seed ydata = pk.gaussian_peaks_integrated( - np.array([[6.0, 15.0, 15.0, 1.2]]), yx + np.array([[6.0, 15.0, 15.0, 1.2, 1.2]]), yx ) - theta0 = np.array([[6.0, 4.0, 4.0, 1.2]]) - lb = np.array([5.9, 3.9, 3.9, 1.1]) - ub = np.array([6.1, 4.1, 4.1, 1.3]) + theta0 = np.array([[6.0, 4.0, 4.0, 1.2, 1.2]]) + lb = np.array([5.9, 3.9, 3.9, 1.1, 1.1]) + ub = np.array([6.1, 4.1, 4.1, 1.3, 1.3]) out = pk._lsq_curvefit(theta0, yx, ydata, lb, ub, max_nfev=50) # stays within the tight bounds (no crash, no bound violation) self.assertTrue(np.all(out.ravel() >= lb - 1e-9)) @@ -182,25 +234,75 @@ def test_non_improving_step_increases_damping(self): class TestIsotropicWarmStart(unittest.TestCase): """Two-stage fitting preserves round peaks while permitting elongation.""" + def test_shared_width_fit(self): + """The constrained LM stage recovers a round peak in five columns.""" + yx = _grid(21, 21) + true = np.array([[5.0, 10.3, 9.7, 0.8, 0.8]]) + theta0 = np.array([[3.0, 10.0, 10.0, 0.5, 0.5]]) + lb, ub = pk._make_bounds(theta0[:, 1:3], 21, 21) + ub[3] = ub[4] + out = pk._lsq_curvefit( + theta0, + yx, + pk.gaussian_peaks_integrated(true, yx), + lb, + ub, + shared_widths=True, + ) + np.testing.assert_allclose(out, true, atol=0.01) + np.testing.assert_array_equal(out[:, 3], out[:, 4]) + + def test_shared_width_constraint_requires_matching_columns(self): + """Tied updates must not silently reinterpret unequal inputs/bounds.""" + for mismatch in ["theta", "lower", "upper"]: + theta = np.array([[1.0, 1.0, 1.0, 0.5, 0.5]]) + lb = np.array([0.0, 0.0, 0.0, 0.35, 0.35]) + ub = np.array([2.0, 2.0, 2.0, 1.0, 1.0]) + if mismatch == "theta": + theta[0, 4] = 0.6 + elif mismatch == "lower": + lb[4] = 0.4 + else: + ub[4] = 0.9 + with self.subTest(mismatch=mismatch): + with self.assertRaisesRegex( + ValueError, "requires equal y/x widths and bounds" + ): + pk._lsq_curvefit( + theta, + _grid(3, 3), + np.zeros(9), + lb, + ub, + shared_widths=True, + ) + def test_stages_and_bounds(self): """Relaxation starts from the fitted shared widths, not the seed.""" theta0 = np.array([[3.0, 10.0, 10.0, 4.0, 0.8]]) original = theta0.copy() lb, ub = pk._make_bounds(theta0[:, 1:3], 21, 21) - iso_fit = np.array([[5.0, 10.3, 9.7, 0.9]]) + iso_fit = np.array([[5.0, 10.3, 9.7, 0.9, 0.9]]) final_fit = np.array([[5.0, 10.3, 9.7, 4.0, 0.7]]) with patch.object( pk, "_lsq_curvefit", side_effect=[iso_fit, final_fit] ) as solver: out = pk._fit_isotropic_then_anisotropic( - theta0, _grid(21, 21), np.zeros(441), lb, ub, + theta0, + _grid(21, 21), + np.zeros(441), + lb, + ub, max_nfev=100, ) self.assertEqual(solver.call_count, 2) first, second = solver.call_args_list - self.assertEqual(first.args[0].shape, (1, 4)) - self.assertEqual(first.args[3][3], 0.35) - self.assertEqual(first.args[4][3], 1.0) + self.assertEqual(first.args[0].shape, (1, 5)) + np.testing.assert_array_equal(first.args[0][:, 3], first.args[0][:, 4]) + np.testing.assert_array_equal(first.args[3][3:], [0.35, 0.35]) + np.testing.assert_array_equal(first.args[4][3:], [1.0, 1.0]) + self.assertTrue(first.kwargs["shared_widths"]) + self.assertNotIn("shared_widths", second.kwargs) np.testing.assert_array_equal( second.args[0], [[5.0, 10.3, 9.7, 0.9, 0.9]] ) @@ -214,14 +316,18 @@ def test_stages_and_bounds(self): def test_recovers_round_and_elongated_peaks(self): """Joint fitting recovers round and tall peaks after relaxation.""" yx = _grid(65, 31) - true = np.array([ - [5.0, 15.3, 8.7, 0.8, 0.8], - [8.0, 40.4, 22.2, 7.0, 0.7], - ]) - theta0 = np.array([ - [3.0, 15.0, 9.0, 0.5, 0.5], - [4.0, 40.0, 22.0, 0.5, 0.5], - ]) + true = np.array( + [ + [5.0, 15.3, 8.7, 0.8, 0.8], + [8.0, 40.4, 22.2, 7.0, 0.7], + ] + ) + theta0 = np.array( + [ + [3.0, 15.0, 9.0, 0.5, 0.5], + [4.0, 40.0, 22.0, 0.5, 0.5], + ] + ) lb, ub = pk._make_bounds(theta0[:, 1:3], 65, 31) out = pk._fit_isotropic_then_anisotropic( theta0, yx, pk.gaussian_peaks_integrated(true, yx), lb, ub @@ -262,11 +368,11 @@ def test_make_bounds(self): def test_peak_mask_and_empty(self): """Peak centers mark the mask; an empty parameter set marks nothing.""" - tf = np.array([[1.0, 3.2, 4.8, 1.0]]) + tf = np.array([[1.0, 3.2, 4.8, 1.0, 1.0]]) mask = pk._peak_mask(tf, 8, 8) self.assertTrue(mask[3, 5]) self.assertEqual(mask.sum(), 1) - empty = pk._peak_mask(np.zeros((0, 4)), 8, 8) + empty = pk._peak_mask(np.zeros((0, 5)), 8, 8) self.assertFalse(empty.any()) def test_buffer_mask(self): @@ -383,8 +489,12 @@ def test_elongated_blob_detected(self): """A tall, narrow blob is fit as one anisotropic Gaussian.""" act = _single_gaussian_plane(65, 25, 32.3, 12.4, 10.0, 7.0, 0.7) out = pk.detect_peaks_2d( - act, np.zeros_like(act, dtype=bool), - mu_bg=0.0, sigma_bg=0.05, peak_thresh=1.0, peak_th=3.0, + act, + np.zeros_like(act, dtype=bool), + mu_bg=0.0, + sigma_bg=0.05, + peak_thresh=1.0, + peak_th=3.0, ) self.assertEqual(out.shape, (1, 5)) np.testing.assert_allclose( diff --git a/tests/test_bandsilo_phase_filter.py b/tests/test_bandsilo_phase_filter.py index 30c6a4d..9d2972c 100644 --- a/tests/test_bandsilo_phase_filter.py +++ b/tests/test_bandsilo_phase_filter.py @@ -4,12 +4,22 @@ import numpy as np -from giant_python.bandsilo import summary_images as si +from giant_python.extraction.band import summary_images as si -def brute_median(image, selected, ref_r, ref_c, ref_d, height, width, - tolerance, minimum): - """Independent small-grid reference using explicit distance/window loops.""" +# Preserve the independent scientific reference's explicit sampling loops. +def brute_median( # noqa: C901 + image, + selected, + ref_r, + ref_c, + ref_d, + height, + width, + tolerance, + minimum, +): + """Compute a small-grid reference with explicit distance/window loops.""" phases = np.full(image.shape, np.nan) med = np.full(image.shape, np.nan) for flat in selected: @@ -24,20 +34,32 @@ def brute_median(image, selected, ref_r, ref_c, ref_d, height, width, if not np.isfinite(phases[z, r, c]) or not np.isfinite(image[z, r, c]): continue samples = [] - for sr in range(max(0, r - height // 2), min(image.shape[1], r + height // 2 + 1)): - for sc in range(max(0, c - width // 2), min(image.shape[2], c + width // 2 + 1)): - if abs(phases[z, sr, sc] - phases[z, r, c]) <= tolerance and np.isfinite(image[z, sr, sc]): + for sr in range( + max(0, r - height // 2), min(image.shape[1], r + height // 2 + 1) + ): + for sc in range( + max(0, c - width // 2), min(image.shape[2], c + width // 2 + 1) + ): + if abs( + phases[z, sr, sc] - phases[z, r, c] + ) <= tolerance and np.isfinite(image[z, sr, sc]): samples.append(image[z, sr, sc]) if len(samples) >= minimum: med[z, r, c] = np.median(samples) first_med = med.copy() for flat in selected: z, r, c = np.unravel_index(flat, image.shape) - if (not np.isfinite(phases[z, r, c]) or not np.isfinite(image[z, r, c]) - or np.isfinite(first_med[z, r, c])): + if ( + not np.isfinite(phases[z, r, c]) + or not np.isfinite(image[z, r, c]) + or np.isfinite(first_med[z, r, c]) + ): continue - window = first_med[z, max(0, r - height // 2):r + height // 2 + 1, - max(0, c - width // 2):c + width // 2 + 1] + window = first_med[ + z, + max(0, r - height // 2) : r + height // 2 + 1, + max(0, c - width // 2) : c + width // 2 + 1, + ] finite = window[np.isfinite(window)] if finite.size: med[z, r, c] = np.mean(finite) @@ -45,14 +67,21 @@ def brute_median(image, selected, ref_r, ref_c, ref_d, height, width, class TestPhaseMatchedFilter(unittest.TestCase): + """Check phase membership, spatial support, and frozen fallback medians.""" + def test_overlapping_curved_bands_match_brute_force(self): + """Match explicit samples across depths, gaps, and chunk sizes.""" rng = np.random.default_rng(8) image = rng.normal(size=(2, 23, 17)).astype(np.float32) # Independent, nonparallel trajectories with overlapping column ranges. cols = np.arange(17) ref_c = np.tile(cols, 4) - ref_r = np.r_[4 + abs(cols - 8) // 2, 16 - abs(cols - 8) // 2, - 3 + cols // 3, 19 - cols // 3].astype(float) + ref_r = np.r_[ + 4 + abs(cols - 8) // 2, + 16 - abs(cols - 8) // 2, + 3 + cols // 3, + 19 - cols // 3, + ].astype(float) ref_d = np.repeat([0, 1], 34) keep = ref_c != 7 # Column gap is not interpolated. ref_r, ref_c, ref_d = ref_r[keep], ref_c[keep], ref_d[keep] @@ -60,18 +89,29 @@ def test_overlapping_curved_bands_match_brute_force(self): image.ravel()[selected[::29]] = np.nan image.ravel()[selected[::41]] = np.inf original = image.copy() - expected = brute_median(image, selected, ref_r, ref_c, ref_d, 9, 7, 0, 3) + expected = brute_median( + image, selected, ref_r, ref_c, ref_d, 9, 7, 0, 3 + ) for chunk_size in (1, 53, 4096): actual = si.phase_matched_nanmedian( - image, selected[::-1], ref_r, ref_c, ref_d, - height=9, width=7, min_samples=3, + image, + selected[::-1], + ref_r, + ref_c, + ref_d, + height=9, + width=7, + min_samples=3, chunk_size=chunk_size, ) - np.testing.assert_allclose(actual, expected, atol=1e-6, equal_nan=True) + np.testing.assert_allclose( + actual, expected, atol=1e-6, equal_nan=True + ) self.assertTrue(np.isnan(actual[:, :, 7]).all()) np.testing.assert_array_equal(image, original) def test_signed_phase_not_absolute_distance(self): + """Keep opposite signed offsets in distinct sample populations.""" image = np.zeros((1, 13, 11), dtype=np.float32) cols = np.arange(11) centers = np.full(11, 6) @@ -83,21 +123,38 @@ def test_signed_phase_not_absolute_distance(self): self.assertEqual(med[0, 7, 5], 100) def test_phase_tolerance_and_minimum_samples(self): + """Match the reference as tolerance and sample thresholds vary.""" image = np.arange(5 * 7, dtype=float).reshape(1, 5, 7) cols = np.arange(7) centers = 2 + 0.25 * (cols % 2) selected = np.arange(image.size) for tolerance in (0, 0.3, 1.0): for minimum in (1, 5, 100): - truth = brute_median(image, selected, centers, cols, np.zeros(7), - 5, 7, tolerance, minimum) + truth = brute_median( + image, + selected, + centers, + cols, + np.zeros(7), + 5, + 7, + tolerance, + minimum, + ) result = si.phase_matched_nanmedian( - image, selected, centers, cols, height=5, width=7, - phase_tolerance=tolerance, min_samples=minimum, + image, + selected, + centers, + cols, + height=5, + width=7, + phase_tolerance=tolerance, + min_samples=minimum, ) np.testing.assert_allclose(result, truth, equal_nan=True) def test_midpoint_rows_and_duplicate_centers_are_retained(self): + """Retain midpoint support while deduplicating reference centers.""" image = np.ones((1, 16, 11)) cols = np.tile(np.arange(11), 3) centers = np.repeat([4, 4, 12], 11) # Deduplicate the top band. @@ -107,21 +164,28 @@ def test_midpoint_rows_and_duplicate_centers_are_retained(self): self.assertTrue(np.isfinite(med[0, 4]).all()) centers = np.repeat([4, 4, 13], 11) med = si.phase_matched_nanmedian(image, selected, centers, cols) - np.testing.assert_array_equal(med[0, 8:10], 1) # Near ties also retained. + np.testing.assert_array_equal( + med[0, 8:10], 1 + ) # Near ties also retained. def test_midpoint_tie_chooses_smaller_reference_row(self): - image = np.array([[[10., 20., 30., 40., 50., 100., 200.]]]) + """Resolve equidistant references toward the smaller row.""" + image = np.array([[[10.0, 20.0, 30.0, 40.0, 50.0, 100.0, 200.0]]]) cols = np.arange(7) # First five pixels are ties with signed phase +4; last two are -4. lower = np.r_[np.full(5, -4), np.full(2, -14)] upper = np.full(7, 4) selected = np.arange(image.size) med = si.phase_matched_nanmedian( - image, selected, np.r_[lower, upper], np.tile(cols, 2), + image, + selected, + np.r_[lower, upper], + np.tile(cols, 2), ) self.assertEqual(med[0, 0, 3], 30) def test_default_exact_phase_does_not_pool_nearby_offsets(self): + """Pool nearby phases only when an explicit tolerance permits it.""" image = np.full((1, 4, 11), np.nan) cols = np.arange(11) centers = np.where(cols % 2, 1.25, 1.0) @@ -131,22 +195,31 @@ def test_default_exact_phase_does_not_pool_nearby_offsets(self): self.assertEqual(med[0, 2, 5], 100) # Five exact +0.75 matches. self.assertEqual(med[0, 2, 4], 10) # Five exact +1 matches. tolerant = si.phase_matched_nanmedian( - image, selected, centers, cols, phase_tolerance=0.5, + image, + selected, + centers, + cols, + phase_tolerance=0.5, ) self.assertEqual(tolerant[0, 2, 5], 10) def test_irregular_spacing_keeps_pixel_offsets_not_fractional_phase(self): + """Measure phase in pixels rather than local inter-band fractions.""" image = np.full((1, 9, 7), np.nan) cols = np.arange(7) gaps = np.array([8, 12, 8, 12, 8, 12, 8]) image[0, 3, :] = np.arange(1, 8) # +1 in every column. refs = np.r_[np.full(7, 2), 2 + gaps] med = si.phase_matched_nanmedian( - image, np.flatnonzero(np.isfinite(image)), refs, np.tile(cols, 2), + image, + np.flatnonzero(np.isfinite(image)), + refs, + np.tile(cols, 2), ) self.assertEqual(med[0, 3, 3], 4) def test_fallback_uses_first_pass_medians_not_raw_or_filled_values(self): + """Freeze first-pass medians so fallback values cannot propagate.""" image = np.full((1, 3, 11), np.nan, dtype=np.float32) image[0, 0] = np.arange(11) ** 2 image[0, 1:3, 5] = 9999 @@ -154,43 +227,63 @@ def test_fallback_uses_first_pass_medians_not_raw_or_filled_values(self): cols = np.arange(11) expected_med = [] for col in cols: - samples = image[0, 0, max(0, col - 5):col + 6] + samples = image[0, 0, max(0, col - 5) : col + 6] median = np.median(samples) expected_med.append(median) for chunk in (1, 4, 4096): med = si.phase_matched_nanmedian( - image, selected[::-1], np.zeros(11), cols, - height=3, chunk_size=chunk, + image, + selected[::-1], + np.zeros(11), + cols, + height=3, + chunk_size=chunk, ) np.testing.assert_allclose(med[0, 0], expected_med) - self.assertAlmostEqual(float(med[0, 1, 5]), np.mean(expected_med), places=5) + self.assertAlmostEqual( + float(med[0, 1, 5]), np.mean(expected_med), places=5 + ) # Filled row 1 cannot propagate into row 2, even in later chunks. self.assertTrue(np.isnan(med[0, 2, 5])) self.assertTrue(np.isnan(med[~np.isfinite(image)]).all()) ordered = si.phase_matched_nanmedian( - image, selected, np.zeros(11), cols, height=3, chunk_size=chunk, + image, + selected, + np.zeros(11), + cols, + height=3, + chunk_size=chunk, ) np.testing.assert_allclose(ordered, med, equal_nan=True) out = si.finalize_activity_image( - image.copy(), selected, np.zeros(selected.size), - np.zeros(11), cols, phase_window_height=3, + image.copy(), + selected, + np.zeros(selected.size), + np.zeros(11), + cols, + phase_window_height=3, ) expected = 9999 - np.mean(expected_med) np.testing.assert_allclose(out[0, 1, 5], expected, rtol=1e-6) self.assertEqual(out[0, 2, 5], 0) # Vertical window hits image edge. def test_empty_fallback_is_nan_without_warnings(self): + """Leave unsupported windows NaN without empty-reduction warnings.""" import warnings image = np.ones((1, 3, 3)) with warnings.catch_warnings(): warnings.simplefilter("error", RuntimeWarning) med = si.phase_matched_nanmedian( - image, np.arange(image.size), np.ones(3), np.arange(3), + image, + np.arange(image.size), + np.ones(3), + np.arange(3), ) self.assertTrue(np.isnan(med).all()) def test_missing_depth_geometry_and_empty_support(self): + """Require multi-plane geometry and preserve absent-plane NaNs.""" image = np.ones((2, 9, 11)) cols, centers = np.arange(11), np.full(11, 4) selected = np.arange(image.size) @@ -208,6 +301,7 @@ def test_missing_depth_geometry_and_empty_support(self): self.assertTrue(np.isnan(med).all()) def test_finalization_masks_samples_and_preserves_motion_geometry(self): + """Use shifted geometry and exclude invalid samples at finalization.""" cols = np.arange(3, 14) centers = 5 + abs(cols - 8) // 2 image = np.arange(20 * 20, dtype=float).reshape(1, 20, 20) @@ -218,32 +312,50 @@ def test_finalization_masks_samples_and_preserves_motion_geometry(self): image_masked = image.copy() image_masked.ravel()[nan_ct > 0.5] = np.nan med = si.phase_matched_nanmedian( - image_masked, selected[nan_ct <= 0.5], centers + dr, cols + dc, + image_masked, + selected[nan_ct <= 0.5], + centers + dr, + cols + dc, ) expected = image_masked - med support = np.isfinite(image_masked) for z, r, c in zip(*np.where(support)): - if (r < 3 or r + 3 >= image.shape[1] - or not support[z, r - 3:r + 4, c].all()): + if ( + r < 3 + or r + 3 >= image.shape[1] + or not support[z, r - 3 : r + 4, c].all() + ): expected[z, r, c] = 0 actual = si.finalize_activity_image( - image.copy(), selected, nan_ct, centers + dr, cols + dc, + image.copy(), + selected, + nan_ct, + centers + dr, + cols + dc, ) np.testing.assert_allclose(actual, expected, equal_nan=True) def test_invalid_parameters(self): + """Reject invalid window, chunk, sample, and tolerance settings.""" image = np.ones((1, 5, 7)) for options in ( - {"height": 4}, {"width": 0}, {"chunk_size": 0}, - {"min_samples": 0}, {"phase_tolerance": -1}, - {"phase_tolerance": np.inf}, {"phase_tolerance": np.nan}, + {"height": 4}, + {"width": 0}, + {"chunk_size": 0}, + {"min_samples": 0}, + {"phase_tolerance": -1}, + {"phase_tolerance": np.inf}, + {"phase_tolerance": np.nan}, ): with self.subTest(options=options), self.assertRaises(ValueError): si.phase_matched_nanmedian( - image, np.arange(image.size), np.full(7, 2), np.arange(7), + image, + np.arange(image.size), + np.full(7, 2), + np.arange(7), **options, ) if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/test_bandsilo_pipeline.py b/tests/test_bandsilo_pipeline.py index a0bc558..83f8f7a 100644 --- a/tests/test_bandsilo_pipeline.py +++ b/tests/test_bandsilo_pipeline.py @@ -1,30 +1,34 @@ -"""Tests for giant_python.bandsilo.pipeline output assembly + writing. - -Covers the pure/testable Phase-8 surface: per-path output assembly, the -byte-compatible ``experiment_summary.h5`` writer, and the returned -``ExperimentSummary`` builder. The full orchestration driver -(``extract_band_sources`` / ``_process_dmd``) reads SLAP2 files, runs -the torch NMF, and does disk IO end-to-end, so it is not unit-tested here (its -Phase 1-7 kernels are each covered/cross-checked in their own modules). +"""Canonical band output assembly, shared writing and localization wiring. + +Tests target the implementation owners rather than compatibility re-exports. +Public driver composition is covered separately in test_band_workflow. """ import os import tempfile import unittest from contextlib import ExitStack -from unittest.mock import patch +from unittest.mock import call, patch import h5py import numpy as np import torch -from giant_python.bandsilo import pipeline as pl +from giant_python.extraction.band import result_assembly as pl +from giant_python.extraction.band import workflow as wf +from giant_python.extraction.band.types import TrialTraceResult +from giant_python.io.experiment_summary import write_summary +from giant_python.models.experiment_summary import ( + FRAME_INFO_KEYS, + ExperimentSummary, +) +from giant_python.models.params import BandSiloParams, ExecutionOptions def _trial(n_frames, n_sources, n_channels, n_soma): - """One synthetic get_high_res_traces 8-tuple.""" + """One synthetic named trial record, matching the trace producer.""" rng = np.random.default_rng(n_frames) - return ( + return TrialTraceResult( rng.random((n_frames, n_sources)).astype(np.float32), # phi (dF_ls) (rng.random((n_frames, n_sources)) + 5).astype(np.float32), # F0 np.arange(n_frames, dtype=float), # frames @@ -82,7 +86,116 @@ def _path_result( class TestLocalizeMedianMotion(unittest.TestCase): """Check median geometry without running the expensive rho/NMF stages.""" + def test_dog_settings_and_all_source_columns_reach_output(self): + """Keep DoG widths, reference centers and source matrices unchanged.""" + ref_d = np.array([0, 0]) + ref_r, ref_c = np.array([1, 2]), np.array([2, 2]) + selected = np.array([0, 1, 2]) + coords = np.zeros((3, 3)) + umyx = np.array([[2, -1]]) + mot_inds = np.zeros(4, dtype=int) + residual = np.ones((2, 4)) + rho = np.ones((3, 4)) + seeds = np.array([[0, 1, 2], [0, 2, 2]]) + fitted = { + "A": torch.ones((12, 2)), + "source_params": torch.ones((2, 6)), + "source_snr": np.ones(2), + } + with ExitStack() as stack: + stack.enter_context( + patch.object( + wf.operators, "build_motion_h_matrices", return_value=["H"] + ) + ) + kernels = stack.enter_context( + patch.object( + wf.operators, + "gaussian_kernel_2d", + side_effect=[("center", (0, 0)), ("surround", (1, 1))], + ) + ) + stack.enter_context( + patch.object( + wf.geo, + "selected_pixels_2d_for_plane", + return_value=(selected, coords), + ) + ) + convolution = stack.enter_context( + patch.object( + wf.operators, + "build_convolution_matrix", + side_effect=["D", "D_exp"], + ) + ) + compute_rho = stack.enter_context( + patch.object(wf.activity, "compute_rho", return_value=rho) + ) + stack.enter_context( + patch.object( + wf.activity, "mask_high_nan_rho", return_value=np.zeros(3) + ) + ) + stack.enter_context(patch.object(wf.activity, "decay_kernel_1d")) + stack.enter_context( + patch.object(wf.activity, "smooth_rho", return_value=rho) + ) + stack.enter_context( + patch.object(wf.si, "accumulate_activity_image") + ) + stack.enter_context(patch.object(wf.si, "finalize_activity_image")) + stack.enter_context( + patch.object(wf, "get_act_im_peaks", return_value=seeds) + ) + fit = stack.enter_context( + patch.object(wf, "fit_sources", return_value=fitted) + ) + result = wf._localize( + residual, + umyx, + mot_inds, + selected, + coords, + 12, + ref_d, + ref_r, + ref_c, + 1, + 3, + 4, + np.ones((3, 3)), + 2, + None, + None, + 10, + BandSiloParams(), + execution=ExecutionOptions(verbose=False), + ) + self.assertEqual(kernels.call_args_list, [call(0.9), call(4.5)]) + self.assertEqual( + convolution.call_args_list[0].args[1:], ("center", (0, 0)) + ) + self.assertEqual( + convolution.call_args_list[1].args[1:], ("surround", (1, 1)) + ) + self.assertIs(compute_rho.call_args.args[7], ref_d) + self.assertIs(compute_rho.call_args.args[8], ref_r) + self.assertIs(compute_rho.call_args.args[9], ref_c) + self.assertIs(fit.call_args.args[0], seeds) + self.assertIs(fit.call_args.args[1], residual) + self.assertIs(fit.call_args.args[3], umyx) + self.assertIs(fit.call_args.args[4], mot_inds) + self.assertIs(fit.call_args.args[5], selected) + self.assertIs(fit.call_args.args[6], coords) + self.assertIs(result[2], fitted["A"]) + self.assertIs(result[3], fitted["source_params"]) + self.assertIs(result[4], fitted["source_snr"]) + np.testing.assert_array_equal(result[1], seeds) + self.assertIsNot(result[1], seeds) + def test_dominant_retained_motion_shifts_only_median_geometry(self): + """Dropped frames do not vote; default run policy preserves centers.""" ref_r = np.array([10, 19]) ref_c = np.array([8, 8]) # Dropped frames outnumber either retained bin and must be ignored. @@ -90,34 +203,50 @@ def test_dominant_retained_motion_shifts_only_median_geometry(self): for displacement in ((2, -5), (-3, 4), (0, 0)): with self.subTest(displacement=displacement), ExitStack() as stack: umyx = np.array([[7, 9], displacement], dtype=float) - for name in ( - "build_motion_h_matrices", "gaussian_kernel_2d", - "selected_pixels_2d_for_plane", "build_convolution_matrix", - "compute_rho", "mask_high_nan_rho", "decay_kernel_1d", - "smooth_rho", + for owner, name in ( + (wf.operators, "build_motion_h_matrices"), + (wf.operators, "gaussian_kernel_2d"), + (wf.geo, "selected_pixels_2d_for_plane"), + (wf.operators, "build_convolution_matrix"), + (wf.activity, "compute_rho"), + (wf.activity, "mask_high_nan_rho"), + (wf.activity, "decay_kernel_1d"), + (wf.activity, "smooth_rho"), ): - stack.enter_context(patch.object( - pl.bg, name, return_value=(None, None) - )) - stack.enter_context(patch.object( - pl.si, "accumulate_activity_image" - )) + stack.enter_context( + patch.object(owner, name, return_value=(None, None)) + ) + stack.enter_context( + patch.object(wf.si, "accumulate_activity_image") + ) # Stop at finalization; peak detection and NMF are unrelated. - finalize = stack.enter_context(patch.object( - pl.si, "finalize_activity_image", - side_effect=RuntimeError("median reached"), - )) + finalize = stack.enter_context( + patch.object( + wf.si, + "finalize_activity_image", + side_effect=RuntimeError("median reached"), + ) + ) with self.assertRaisesRegex(RuntimeError, "median reached"): - pl._localize( + wf._localize( residual=np.zeros((2, mot_inds.size)), - umyx=umyx, mot_inds_yx=mot_inds, - sel_pix_idxs=np.array([0]), pixel_coords=None, - n_pixels=1600, ref_d=np.array([0, 0]), - ref_r=ref_r, ref_c=ref_c, num_fast_zs=1, - dmd_pixels_per_column=40, dmd_pixels_per_row=40, - psf2d=np.ones((3, 3)), num_super_pixels=2, - sparse_h_inds=None, sparse_h_vals=None, align_hz=10, - params=pl.SiloParams(verbose=False), + umyx=umyx, + mot_inds_yx=mot_inds, + sel_pix_idxs=np.array([0]), + pixel_coords=None, + n_pixels=1600, + ref_d=np.array([0, 0]), + ref_r=ref_r, + ref_c=ref_c, + num_fast_zs=1, + dmd_pixels_per_column=40, + dmd_pixels_per_row=40, + psf2d=np.ones((3, 3)), + num_super_pixels=2, + sparse_h_inds=None, + sparse_h_vals=None, + align_hz=10, + params=BandSiloParams(), ) finalize.assert_called_once() np.testing.assert_array_equal( @@ -133,6 +262,133 @@ def test_dominant_retained_motion_shifts_only_median_geometry(self): np.testing.assert_array_equal(ref_c, [8, 8]) +class TestStageComposition(unittest.TestCase): + """Check background wiring and empty localization without sparse fits.""" + + def test_default_execution_skips_empty_planes_and_preserves_grid(self): + """Interpolate selected planes; baseline and assembly share data.""" + data = np.array([[3.0, 4.0], [5.0, 6.0]], dtype=np.float32) + selected = np.array([4, 5]) + ref_d, ref_r, ref_c = ( + np.array([1, 1]), + np.array([0, 1]), + np.array([0, 1]), + ) + umyx, mot_inds = np.array([[0, 0]]), np.array([0, 0]) + plane_pixels = np.array([[0, 0], [0, 1]]) + interpolated = data + 10 + baseline, assembled = data + 20, data + 30 + params = BandSiloParams(baseline_window_s=0.5) + with ExitStack() as stack: + expand = stack.enter_context( + patch.object(wf.operators, "expand_psf") + ) + planes = stack.enter_context( + patch.object( + wf.geo, + "selected_pixels_2d_for_plane", + side_effect=[ + (np.array([], dtype=int), np.empty((0, 2))), + (np.array([0, 1]), plane_pixels), + ], + ) + ) + interpolate = stack.enter_context( + patch.object( + wf.bg, + "build_interp_data", + return_value=(interpolated, None, None), + ) + ) + rolling = stack.enter_context( + patch.object( + wf.bg, "compute_rolling_baseline", return_value=baseline + ) + ) + assemble = stack.enter_context( + patch.object( + wf.bg, "assemble_background", return_value=assembled + ) + ) + psf = np.ones((3, 3)) + result = wf._estimate_background( + data, + umyx, + mot_inds, + selected, + ref_d, + ref_r, + ref_c, + 2, + 2, + 2, + psf, + 2, + 10, + params, + ) + expand.assert_called_once_with(psf) + self.assertEqual(planes.call_count, 2) + interpolate.assert_called_once() + np.testing.assert_array_equal(interpolate.call_args.args[0], data) + np.testing.assert_array_equal(interpolate.call_args.args[1], ref_r) + np.testing.assert_array_equal(interpolate.call_args.args[2], ref_c) + self.assertIs(interpolate.call_args.args[3], plane_pixels) + self.assertEqual( + interpolate.call_args.kwargs["method"], + params.background_interpolation, + ) + rolling.assert_called_once() + np.testing.assert_array_equal(rolling.call_args.args[0], interpolated) + self.assertEqual(rolling.call_args.args[0].dtype, np.float32) + self.assertEqual(rolling.call_args.args[1], 5) + assemble.assert_called_once() + self.assertIs(assemble.call_args.args[0], baseline) + self.assertIs(assemble.call_args.args[1], umyx) + self.assertIs(assemble.call_args.args[2], mot_inds) + self.assertEqual(assemble.call_args.args[7:], (2, 2, 2, 2)) + self.assertIs(result, assembled) + + def test_no_retained_motion_bins_do_not_invent_a_source_fit(self): + """Characterize the existing empty argmax error without a fallback.""" + with ExitStack() as stack: + for owner, name in ( + (wf.operators, "build_motion_h_matrices"), + (wf.activity, "compute_rho"), + (wf.activity, "mask_high_nan_rho"), + (wf.activity, "smooth_rho"), + (wf.si, "accumulate_activity_image"), + ): + stack.enter_context(patch.object(owner, name)) + finalize = stack.enter_context( + patch.object(wf.si, "finalize_activity_image") + ) + fit = stack.enter_context(patch.object(wf, "fit_sources")) + with self.assertRaisesRegex(ValueError, "argmax.*empty"): + wf._localize( + np.ones((2, 3)), + np.empty((0, 2)), + np.full(3, -1), + np.array([], dtype=int), + np.empty((0, 3)), + 4, + np.array([0, 0]), + np.array([0, 1]), + np.array([0, 1]), + 0, + 2, + 2, + np.ones((3, 3)), + 2, + None, + None, + 10, + BandSiloParams(), + ) + finalize.assert_not_called() + fit.assert_not_called() + + class TestDmdUserRois(unittest.TestCase): """_dmd_user_rois slices a resolved session selection per DMD.""" @@ -178,29 +434,38 @@ def test_shape_and_channels(self): self.assertTrue(np.all(np.isnan(out[:, 1, :]))) -class TestAssemblePathOutputs(unittest.TestCase): - """assemble_path_outputs shapes and content.""" +class TestAssemblePathSummary(unittest.TestCase): + """Typed path assembly preserves numerical shapes and content.""" def test_shapes(self): """Every dataset has the schema shape.""" pr = _path_result(n_sources=2, n_channels=1, nz=1, npc=4, npr=5) - asm = pl.assemble_path_outputs(pr) - self.assertEqual(asm["profiles"].shape, (2, 1, 4, 5)) - self.assertEqual(asm["coords"].shape, (2, 3)) - self.assertEqual(asm["dF_ls"].shape, (2, 1, 13)) # 6+7 frames - self.assertEqual(asm["F0"].shape, (2, 1, 13)) - self.assertEqual(asm["SNR"].shape, (2, 1)) - self.assertEqual(asm["global_f"].shape, (1, 13)) - self.assertEqual(list(asm["trial_num_frames"].ravel()), [6, 7]) - self.assertEqual(asm["discard_frames"].shape, (13, 1)) - self.assertIsNone(asm["user_rois"]) + path = pl.assemble_path_summary(pr) + self.assertEqual(len(path.sources), 2) + for source in path.sources: + self.assertEqual(source.profile.shape, (1, 4, 5)) + self.assertEqual(source.coords.shape, (3,)) + self.assertEqual(source.df_ls.shape, (1, 13)) + self.assertEqual(source.f0.shape, (1, 13)) + self.assertEqual(source.df_ls.dtype, np.float64) + self.assertEqual(source.f0.dtype, np.float64) + np.testing.assert_array_equal([s.snr for s in path.sources], [1, 2]) + self.assertEqual(path.global_f.shape, (1, 13)) + self.assertEqual( + list(path.frame_info.trial_num_frames.ravel()), [6, 7] + ) + self.assertEqual(path.frame_info.discard_frames.shape, (13, 1)) + self.assertEqual(path.user_rois, []) + self.assertFalse(path.annotation_enabled) + self.assertIs(path.z_depths, pr.z_depths) def test_two_channels_global_shape(self): """Two channels widen the global-F channel axis.""" pr = _path_result(n_channels=2) - asm = pl.assemble_path_outputs(pr) - self.assertEqual(asm["global_f"].shape, (2, 13)) - self.assertEqual(asm["dF_ls"].shape[1], 2) + path = pl.assemble_path_summary(pr) + self.assertEqual(path.global_f.shape, (2, 13)) + self.assertEqual(path.sources[0].df_ls.shape, (2, 13)) + self.assertTrue(np.isnan(path.sources[0].df_ls[1]).all()) def test_user_rois_assembled(self): """draw_user_rois with masks assembles the user-ROI datasets.""" @@ -211,34 +476,60 @@ def test_user_rois_assembled(self): soma_labels=["SOMA", "SOMA2"], n_soma=2, ) - asm = pl.assemble_path_outputs(pr) - self.assertIsNotNone(asm["user_rois"]) - self.assertEqual(asm["user_rois"]["mask"].shape, (2, 1, 4, 5)) - # F: (rois, channels, total_frames) - self.assertEqual(asm["user_rois"]["F"].shape, (2, 1, 13)) + path = pl.assemble_path_summary(pr) + self.assertTrue(path.annotation_enabled) + self.assertEqual(len(path.user_rois), 2) + for index, roi in enumerate(path.user_rois): + np.testing.assert_array_equal(roi.mask, masks[index]) + self.assertEqual(roi.f.shape, (1, 13)) + self.assertEqual(roi.f.dtype, np.float32) + np.testing.assert_array_equal( + roi.f, + np.concatenate( + [r.user_roi_f[:, index] for r in pr.trace_results] + ).T, + ) def test_user_rois_empty_masklist(self): """draw_user_rois with an empty mask list yields a 0-ROI mask array.""" pr = _path_result( draw_user_rois=True, soma_masks=[], soma_labels=[], n_soma=0 ) - asm = pl.assemble_path_outputs(pr) - self.assertEqual(asm["user_rois"]["mask"].shape, (0, 1, 4, 5)) + path = pl.assemble_path_summary(pr) + self.assertTrue(path.annotation_enabled) + self.assertEqual(path.user_rois, []) + self.assertEqual(path._empty_roi_arrays["mask"].shape, (0, 1, 4, 5)) + self.assertEqual(path._empty_roi_arrays["F"].shape, (0, 1, 13)) + + def test_empty_sources_preserve_geometry_and_frame_data(self): + """No sources still produces shaped empty tensors and real globals.""" + pr = _path_result(n_sources=0, n_channels=2) + path = pl.assemble_path_summary(pr) + self.assertEqual(path.sources, []) + self.assertEqual( + path._empty_source_arrays["profiles"].shape, (0, 1, 4, 5) + ) + self.assertEqual(path._empty_source_arrays["dF_ls"].shape, (0, 2, 13)) + self.assertEqual(path._empty_source_arrays["SNR"].dtype, np.float32) + self.assertEqual(path.global_f.shape, (2, 13)) + self.assertFalse(path.frame_info.discard_frames.any()) class TestWriteExperimentSummary(unittest.TestCase): - """write_experiment_summary produces the byte-compatible schema.""" + """write_summary produces the MATLAB-compatible schema.""" def _write_and_read(self, assembled_paths, params=None): """Write then reopen, returning the h5 file handle's data as dict.""" params = params or {"operator": "test", "numChannels": 1} path = tempfile.mktemp(suffix=".h5") - pl.write_experiment_summary(path, params, assembled_paths) + write_summary( + ExperimentSummary(paths=assembled_paths, params=params), path + ) return path def test_schema_and_values(self): """Root flag/params + one Path group with the expected datasets.""" - asm = pl.assemble_path_outputs(_path_result()) + asm = pl.assemble_path_summary(_path_result()) path = self._write_and_read([asm]) try: with h5py.File(path, "r") as f: @@ -252,7 +543,9 @@ def test_schema_and_values(self): grp["sources/temporal/dF_ls"].shape, (2, 1, 13) ) self.assertEqual(grp["global/F"].shape, (1, 13)) - for key in pl._FRAME_INFO_KEYS: + np.testing.assert_array_equal(grp["Z_depths"], asm.z_depths) + self.assertNotIn("z_depths", grp) + for key in FRAME_INFO_KEYS: self.assertIn(key, grp["frame_info"]) self.assertIn("act_im", grp["visualizations"]) self.assertNotIn("user_rois", grp) @@ -268,7 +561,7 @@ def test_user_rois_written(self): soma_labels=["SOMA"], n_soma=1, ) - path = self._write_and_read([pl.assemble_path_outputs(pr)]) + path = self._write_and_read([pl.assemble_path_summary(pr)]) try: with h5py.File(path, "r") as f: ur = f["Path1/user_rois"] @@ -280,11 +573,13 @@ def test_user_rois_written(self): def test_multi_path_and_overwrite(self): """Two paths write Path1/Path2; re-writing overwrites in place.""" - asm = [pl.assemble_path_outputs(_path_result()) for _ in range(2)] + asm = [pl.assemble_path_summary(_path_result(), i) for i in range(2)] path = self._write_and_read(asm) try: # second write to the same file overwrites the Path groups - pl.write_experiment_summary(path, {"operator": "test"}, asm) + write_summary( + ExperimentSummary(paths=asm, params={"operator": "test"}), path + ) with h5py.File(path, "r") as f: self.assertIn("Path1", f) self.assertIn("Path2", f) @@ -292,21 +587,38 @@ def test_multi_path_and_overwrite(self): os.remove(path) -class TestBuildExperimentSummary(unittest.TestCase): - """build_experiment_summary populates the returned dataclass.""" +class TestExperimentSummary(unittest.TestCase): + """Construct summaries directly from numerical path models.""" + + def test_all_path_fields_are_retained(self): + """Paths retain distinct images, frame metadata and globals.""" + assembled = [ + pl.assemble_path_summary(_path_result(), i) for i in range(2) + ] + assembled[0].z_depths = np.array([[10]]) + assembled[1].z_depths = np.array([[20]]) + assembled[0].global_f[:] = 3 + assembled[1].global_f[:] = 7 + summary = ExperimentSummary(paths=assembled, params={}) + self.assertEqual(len(summary.paths), 2) + for index, path in enumerate(summary.paths): + self.assertEqual(path.path_index, index) + np.testing.assert_array_equal(path.z_depths, [[10 + 10 * index]]) + np.testing.assert_array_equal(path.global_f, 3 + 4 * index) + self.assertIs(path, assembled[index]) def test_sources_and_visualizations(self): """Per-path sources are populated with profile/coords/traces/SNR.""" - asm = pl.assemble_path_outputs(_path_result(n_sources=2)) - summary = pl.build_experiment_summary({"operator": "t"}, [asm]) - self.assertEqual(len(summary.sources), 1) - self.assertEqual(len(summary.sources[0]), 2) - src = summary.sources[0][0] + asm = pl.assemble_path_summary(_path_result(n_sources=2)) + summary = ExperimentSummary(paths=[asm], params={"operator": "t"}) + self.assertEqual(len(summary.paths), 1) + self.assertEqual(len(summary.paths[0].sources), 2) + src = summary.paths[0].sources[0] self.assertEqual(src.profile.shape, (1, 4, 5)) self.assertEqual(src.snr, 1.0) - self.assertIsNotNone(summary.visualizations.act_im) - self.assertEqual(summary.user_rois, [[]]) - self.assertIn("trial_num_frames", summary.frame_info) + self.assertIsNotNone(summary.paths[0].visualizations.act_im) + self.assertEqual(summary.paths[0].user_rois, []) + self.assertIsNotNone(summary.paths[0].frame_info.trial_num_frames) def test_user_rois_populated(self): """User ROIs appear in the summary when present.""" @@ -317,10 +629,10 @@ def test_user_rois_populated(self): soma_labels=["SOMA"], n_soma=1, ) - asm = pl.assemble_path_outputs(pr) - summary = pl.build_experiment_summary({"operator": "t"}, [asm]) - self.assertEqual(len(summary.user_rois[0]), 1) - self.assertEqual(summary.user_rois[0][0].label, "SOMA") + asm = pl.assemble_path_summary(pr) + summary = ExperimentSummary(paths=[asm], params={"operator": "t"}) + self.assertEqual(len(summary.paths[0].user_rois), 1) + self.assertEqual(summary.paths[0].user_rois[0].label, "SOMA") if __name__ == "__main__": diff --git a/tests/test_bandsilo_progress.py b/tests/test_bandsilo_progress.py index 3a28b89..056c47c 100644 --- a/tests/test_bandsilo_progress.py +++ b/tests/test_bandsilo_progress.py @@ -1,10 +1,10 @@ -"""Tests for giant_python.bandsilo.progress (opt-in logging + bars).""" +"""Tests for giant_python.progress (opt-in logging + bars).""" import contextlib import io import unittest -from giant_python.bandsilo.progress import log, progress +from giant_python.progress import log, progress class TestLog(unittest.TestCase): diff --git a/tests/test_bandsilo_summary_images.py b/tests/test_bandsilo_summary_images.py index aa3fec4..1e48781 100644 --- a/tests/test_bandsilo_summary_images.py +++ b/tests/test_bandsilo_summary_images.py @@ -1,4 +1,4 @@ -"""Tests for giant_python.bandsilo.summary_images (Phase 4). +"""Tests for giant_python.extraction.band.summary_images (Phase 4). Exercises the mean image, the spatio-temporal local-maxima activity-image accumulation, and the median-subtraction finalize on small synthetic inputs. @@ -9,7 +9,7 @@ import numpy as np -from giant_python.bandsilo import summary_images as si +from giant_python.extraction.band import summary_images as si class TestMeanImage(unittest.TestCase): @@ -137,6 +137,7 @@ class TestFinalizeActivityImage(unittest.TestCase): """Activity-image masking and local median subtraction in raw units.""" def test_vertical_window_requires_full_valid_support(self): + """Suppress windows crossing edges or invalid vertical support.""" image = np.ones((2, 19, 13), dtype=np.float32) selected_mask = np.ones(image.shape, dtype=bool) selected_mask[0, :4] = False @@ -150,19 +151,29 @@ def test_vertical_window_requires_full_valid_support(self): support = selected_mask & np.isfinite(image) support[0, 9, 6] = False for height in (1, 3, 7): - geometry = dict(ref_r=np.array([9, 9]), ref_c=np.array([6, 6]), - ref_d=np.array([0, 1]), phase_window_height=height) + geometry = dict( + ref_r=np.array([9, 9]), + ref_c=np.array([6, 6]), + ref_d=np.array([0, 1]), + phase_window_height=height, + ) # A fixed median isolates final edge suppression. with patch.object( - si, "phase_matched_nanmedian", return_value=np.zeros_like(image) + si, + "phase_matched_nanmedian", + return_value=np.zeros_like(image), ): out = si.finalize_activity_image( - image.copy(), selected, nan_ct, **geometry, + image.copy(), + selected, + nan_ct, + **geometry, ) for z, r, c in zip(*np.where(support)): lo, hi = r - height // 2, r + height // 2 + 1 fits = ( - lo >= 0 and hi <= image.shape[1] + lo >= 0 + and hi <= image.shape[1] and support[z, lo:hi, c].all() ) self.assertEqual(out[z, r, c], 1 if fits else 0) @@ -172,17 +183,22 @@ def test_vertical_window_requires_full_valid_support(self): self.assertEqual(out[1, 9, 0], 1) # Horizontal edges survive. def test_median_subtraction_preserves_mask(self): + """Subtract the median without restoring excluded pixels.""" image = np.ones((1, 13, 17), dtype=np.float32) image[0, 6, 8] = 7 selected = np.arange(1, image.size) nan_ct = np.zeros(selected.size) nan_ct[-1] = 0.9 for geometry in ( - {}, {"ref_r": np.full(17, 6), "ref_c": np.arange(17)}, + {}, + {"ref_r": np.full(17, 6), "ref_c": np.arange(17)}, ): with self.subTest(snake=bool(geometry)): out = si.finalize_activity_image( - image.copy(), selected, nan_ct, **geometry, + image.copy(), + selected, + nan_ct, + **geometry, ) self.assertEqual(out[0, 6, 8], 6) self.assertEqual(out[0, 6, 7], 0) @@ -194,13 +210,16 @@ def test_median_subtraction_preserves_mask(self): np.testing.assert_allclose(out, expected, equal_nan=True) def test_constant_neighborhoods_are_zero_and_spike_is_preserved(self): - """Subtract constant backgrounds without masking valid neighborhoods.""" + """Subtract constant backgrounds without masking valid neighbors.""" num_fast_zs, npc, npr = 1, 13, 13 act_im = np.ones((num_fast_zs, npc, npr), dtype=np.float32) # all pixels selected and valid (nan_ct = 0) sel_pix_idxs = np.arange(npc * npr) nan_ct = np.zeros(sel_pix_idxs.size) - for geometry in ({}, {"ref_r": np.full(npr, 6), "ref_c": np.arange(npr)}): + for geometry in ( + {}, + {"ref_r": np.full(npr, 6), "ref_c": np.arange(npr)}, + ): for spike in (0, 10): with self.subTest(geometry=bool(geometry), spike=spike): image = act_im.copy() @@ -217,9 +236,11 @@ def test_constant_neighborhoods_are_zero_and_spike_is_preserved(self): def test_high_nan_pixels_masked(self): """Pixels with nan_ct > 0.5 are excluded and set to NaN.""" - num_fast_zs, npc, npr = 1, 13, 13 + npc, npr = 13, 13 act_im = np.arange(npc * npr, dtype=np.float32).reshape(1, npc, npr) - rows, cols = np.meshgrid(np.arange(2, 5), np.arange(2, 8), indexing="ij") + rows, cols = np.meshgrid( + np.arange(2, 5), np.arange(2, 8), indexing="ij" + ) sel_pix_idxs = (rows * npr + cols).ravel() nan_ct = np.zeros(sel_pix_idxs.size) nan_ct[sel_pix_idxs == 3 * npr + 4] = 0.9 @@ -261,18 +282,20 @@ def test_median_follows_motion_shifted_bend(self): act_im[0, row - 1 : row + 2, col] = 10.0 sel_pix_idxs = np.arange(act_im.size) med = si.snake_aligned_nanmedian( - act_im, sel_pix_idxs, - ref_r=center_rows + dr, ref_c=cols + dc, + act_im, + sel_pix_idxs, + ref_r=center_rows + dr, + ref_c=cols + dc, ) self.assertAlmostEqual(float(med[0, 8 + dr, 16 + dc]), 10.0) if dc: unshifted = si.snake_aligned_nanmedian( - act_im, sel_pix_idxs, - ref_r=center_rows, ref_c=cols, - ) - self.assertLess( - float(unshifted[0, 8 + dr, 16 + dc]), 10.0 + act_im, + sel_pix_idxs, + ref_r=center_rows, + ref_c=cols, ) + self.assertLess(float(unshifted[0, 8 + dr, 16 + dc]), 10.0) def test_trajectory_does_not_jump_when_outer_snake_is_missing(self): """Changing snake count does not move the shared trajectory.""" @@ -295,8 +318,12 @@ def test_trajectory_does_not_jump_when_outer_snake_is_missing(self): self.assertAlmostEqual(float(med[0, 19, 4]), 10.0, places=5) def test_median_subtraction_preserves_raw_units_and_scaling(self): - """Both paths subtract their footprint median and retain input scaling.""" - image = np.broadcast_to(np.arange(17), (1, 17, 17)).astype(np.float32).copy() + """Both paths subtract footprint medians and retain input scaling.""" + image = ( + np.broadcast_to(np.arange(17), (1, 17, 17)) + .astype(np.float32) + .copy() + ) image[0, 8, 8] += 6 selected = np.arange(image.size) for geometry, height in ( @@ -313,20 +340,29 @@ def test_median_subtraction_preserves_raw_units_and_scaling(self): float(out[0, 8, 8]), float(image[0, 8, 8] - median) ) rescaled = si.finalize_activity_image( - 4 * image + 13, selected, np.zeros(selected.size), **geometry + 4 * image + 13, + selected, + np.zeros(selected.size), + **geometry, ) np.testing.assert_allclose(4 * out, rescaled, equal_nan=True) def test_median_subtraction_follows_curved_footprint(self): + """Subtract the phase-matched median at a curved band center.""" cols = np.arange(17) ref_r = np.abs(cols - 8) + 5 image = np.zeros((1, 25, 17), dtype=np.float32) for col, row in zip(cols, ref_r): - image[0, row - 2 : row + 3, col] = 20 + col - 8 + 100 * np.arange(-2, 3) + image[0, row - 2 : row + 3, col] = ( + 20 + col - 8 + 100 * np.arange(-2, 3) + ) image[0, 5, 8] = 25 out = si.finalize_activity_image( - image, np.arange(image.size), np.zeros(image.size), - ref_r=ref_r, ref_c=cols, + image, + np.arange(image.size), + np.zeros(image.size), + ref_r=ref_r, + ref_c=cols, ) # The default 7-row window reaches center-phase samples in columns # 5:12 at this bend. Replacing 20 with 25 gives median=21. @@ -337,6 +373,7 @@ class TestSnakeMedian(unittest.TestCase): """Medians use the exact curved, clipped, section-limited footprint.""" def test_matches_explicit_samples_across_chunks(self): + """Match clipped trajectory samples independently of chunk size.""" rng = np.random.default_rng(42) image = rng.normal(size=(2, 20, 17)).astype(np.float32) image[:, 4, ::3] = np.nan @@ -347,7 +384,11 @@ def test_matches_explicit_samples_across_chunks(self): original = image.copy() for height in (3, 5): med = si.snake_aligned_nanmedian( - image, selected, ref_r, cols, height=height, + image, + selected, + ref_r, + cols, + height=height, chunk_size=2, ) for depth, row, col in points: @@ -362,21 +403,30 @@ def test_matches_explicit_samples_across_chunks(self): for offset in range(-(height // 2), height // 2 + 1): sample_row = row + delta + offset if 0 <= sample_row < image.shape[1]: - samples.append(image[depth, sample_row, sample_col]) + samples.append( + image[depth, sample_row, sample_col] + ) expected_med = np.nanmedian(samples) - self.assertAlmostEqual(float(med[depth, row, col]), float(expected_med)) + self.assertAlmostEqual( + float(med[depth, row, col]), float(expected_med) + ) unselected = np.ones(image.size, dtype=bool) unselected[selected] = False self.assertTrue(np.all(np.isnan(med.ravel()[unselected]))) for chunk_size in (1, 4096): other_chunk = si.snake_aligned_nanmedian( - image, selected, ref_r, cols, height=height, + image, + selected, + ref_r, + cols, + height=height, chunk_size=chunk_size, ) np.testing.assert_allclose(med, other_chunk, equal_nan=True) np.testing.assert_array_equal(image, original) def test_empty_and_all_nan_support(self): + """Preserve NaNs for empty output selections and missing samples.""" image = np.full((1, 5, 11), np.nan, dtype=np.float32) cols = np.arange(11) for selected in (np.array([], dtype=int), np.array([27])): diff --git a/tests/test_bandsilo_traces.py b/tests/test_bandsilo_traces.py index 74a758b..efeddbd 100644 --- a/tests/test_bandsilo_traces.py +++ b/tests/test_bandsilo_traces.py @@ -1,4 +1,4 @@ -"""Tests for giant_python.bandsilo.traces (Phase 7 high-res traces). +"""Tests for canonical band traces (Phase 7 high-res traces). Covers the per-trial compute (motion interp/binning, per-motion least-squares ``phi``/``F0``, global and per-ROI fluorescence) and the IO wrapper's @@ -12,8 +12,9 @@ import numpy as np import torch -from giant_python.bandsilo import geometry as geo -from giant_python.bandsilo import traces as tr +from giant_python.extraction.band import operators +from giant_python.extraction.band import traces as tr +from giant_python.extraction.band.types import TrialTraceResult def _geometry(npc=15, npr=15, num_fast_zs=2): @@ -27,7 +28,7 @@ def _geometry(npc=15, npr=15, num_fast_zs=2): smi = np.array(rows, dtype=np.int32) yy, xx = np.mgrid[-2:3, -2:3] psf2d = np.exp(-(yy**2 + xx**2) / 2.0).astype(np.float32) - sh_inds, sh_vals = geo.build_sparse_h(smi, psf2d, npc, npr) + sh_inds, sh_vals = operators.build_sparse_h(smi, psf2d, npc, npr) return dict( npc=npc, npr=npr, @@ -120,6 +121,15 @@ def test_single_channel_shapes_and_fit(self): """Returns the 8-tuple; phi/F0 are per-frame per-source with fits.""" inp = _trial_inputs(num_channels=1) out = _compute(inp) + self.assertIsInstance(out, TrialTraceResult) + self.assertIs(out.d_f, out[0]) + self.assertIs(out.f0_ls, out[1]) + self.assertIs(out.frame_line_idxs, out[2]) + self.assertIs(out.selected_pixels, out[3]) + self.assertIs(out.global_f, out[4]) + self.assertIs(out.reference_offsets, out[5]) + self.assertIs(out.online_shifts, out[6]) + self.assertIs(out.user_roi_f, out[7]) phi, f0, frames, sel, global_f, motion, online, f_soma = out n_sources = inp["a_final"].shape[1] self.assertEqual(phi.shape, (inp["n_frames"], n_sources)) @@ -183,6 +193,7 @@ def test_skipped_trial_returns_empty(self): 2, inp["soma_sps"], ) + self.assertIsInstance(out, TrialTraceResult) self.assertEqual(out[0].shape, (0, inp["a_final"].shape[1])) self.assertEqual(out[4].shape, (0, 2)) # globalF (0, channels) self.assertEqual(out[2].shape, (0,)) # frames diff --git a/tests/test_bandsilo_trial_data.py b/tests/test_bandsilo_trial_data.py index b034b65..0dd9ae9 100644 --- a/tests/test_bandsilo_trial_data.py +++ b/tests/test_bandsilo_trial_data.py @@ -1,4 +1,4 @@ -"""Tests for giant_python.bandsilo.trial_data (pure + mocked-reader paths).""" +"""Tests for canonical band trial reduction and shared numerical helpers.""" import os import tempfile @@ -6,7 +6,8 @@ import numpy as np -from giant_python.bandsilo import trial_data as td +from giant_python.extraction.band import trial_data as td +from giant_python.numerics import interpolation, morphology class _MockDataFile: @@ -50,14 +51,16 @@ class TestNearestInterp(unittest.TestCase): def test_single_point(self): """A single sample returns yp unchanged.""" yp = np.array([7.0]) - out = td.nearest_interp(np.array([0.0, 5.0]), np.array([2.0]), yp) + out = interpolation.nearest_interp( + np.array([0.0, 5.0]), np.array([2.0]), yp + ) np.testing.assert_array_equal(out, yp) def test_midpoint_rounding(self): """Query points snap to the nearest sample boundary.""" xp = np.array([0.0, 10.0]) yp = np.array([1.0, 2.0]) - out = td.nearest_interp(np.array([4.0, 6.0]), xp, yp) + out = interpolation.nearest_interp(np.array([4.0, 6.0]), xp, yp) np.testing.assert_array_equal(out, np.array([1.0, 2.0])) @@ -68,7 +71,7 @@ def test_3x3_fast_path(self): """A single point dilates to its 8-neighborhood.""" mask = np.zeros((5, 5), dtype=bool) mask[2, 2] = True - out = td.fast_dilation(mask, np.ones((3, 3), np.uint8)) + out = morphology.fast_dilation(mask, np.ones((3, 3), np.uint8)) self.assertEqual(out[1:4, 1:4].sum(), 9) self.assertEqual(out.sum(), 9) @@ -76,7 +79,7 @@ def test_leading_axes_fast_path(self): """The fast path dilates only the trailing two axes.""" mask = np.zeros((2, 5, 5), dtype=bool) mask[0, 2, 2] = True - out = td.fast_dilation(mask, np.ones((3, 3), np.uint8)) + out = morphology.fast_dilation(mask, np.ones((3, 3), np.uint8)) self.assertEqual(out[0].sum(), 9) self.assertEqual(out[1].sum(), 0) @@ -84,7 +87,7 @@ def test_generic_cv2_path(self): """A non-3x3 kernel routes through the cv2 fallback.""" mask = np.zeros((5, 5), dtype=bool) mask[2, 2] = True - out = td.fast_dilation(mask, np.ones((1, 3), np.uint8)) + out = morphology.fast_dilation(mask, np.ones((1, 3), np.uint8)) # 1x3 horizontal kernel dilates left/right only. self.assertTrue(out[2, 1] and out[2, 3] and out[2, 2]) self.assertFalse(out[1, 2] or out[3, 2]) @@ -93,7 +96,7 @@ def test_default_kernel(self): """A None kernel uses the 7x7 default (generic path).""" mask = np.zeros((9, 9), dtype=bool) mask[4, 4] = True - out = td.fast_dilation(mask) + out = morphology.fast_dilation(mask) self.assertEqual(out[1:8, 1:8].sum(), 49) @@ -199,9 +202,9 @@ def test_full_read_with_monkeypatched_open(self): "fn_adata": np.array([["a.h5"]], dtype=object), } - orig_open = td._open_slap2_file + orig_open = td.open_slap2_file orig_align = td.load_alignment_data_h5 - td._open_slap2_file = lambda path: df + td.open_slap2_file = lambda path: df td.load_alignment_data_h5 = lambda path: { "motionDSr": np.zeros(2), "motionDSc": np.zeros(2), @@ -220,7 +223,7 @@ def test_full_read_with_monkeypatched_open(self): all_channels=False, ) finally: - td._open_slap2_file = orig_open + td.open_slap2_file = orig_open td.load_alignment_data_h5 = orig_align self.assertIsNotNone(out) diff --git a/tests/test_cli.py b/tests/test_cli.py index 5a43065..a11a788 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,5 +1,8 @@ """Tests for the giant_python.cli argument parser and dispatch.""" +import io +import subprocess +import sys import unittest from unittest import mock @@ -7,7 +10,7 @@ class TestBuildParser(unittest.TestCase): - """build_parser wires the four subcommands and their options.""" + """build_parser exposes only annotation and extraction with valid modes.""" def setUp(self): """Build a fresh parser for each test.""" @@ -18,18 +21,30 @@ def test_no_command(self): args = self.parser.parse_args([]) self.assertIsNone(args.command) - def test_organize(self): - """organize captures the data and save dirs.""" - args = self.parser.parse_args(["organize", "/data", "/results"]) - self.assertEqual(args.command, "organize") - self.assertEqual(args.data_dir, "/data") - self.assertEqual(args.save_dir, "/results") + def test_removed_commands_are_absent(self): + """Removed stages have neither CLI help entries nor handlers.""" + help_text = self.parser.format_help() + for command in ("annotate", "extract"): + self.assertIn(command, help_text) + for command in ("organize", "register"): + with self.subTest(command=command): + self.assertNotIn(command, help_text) + self.assertFalse(hasattr(cli, "_cmd_" + command)) + with mock.patch( + "sys.stderr", new_callable=io.StringIO + ) as stderr: + with self.assertRaises(SystemExit) as error: + self.parser.parse_args([command]) + self.assertEqual(error.exception.code, 2) + self.assertIn("invalid choice", stderr.getvalue()) + self.assertIn(command, stderr.getvalue()) def test_annotate_defaults(self): """annotate defaults to band scan, no draw, auto interactivity.""" args = self.parser.parse_args(["annotate", "tt.h5"]) self.assertEqual(args.command, "annotate") self.assertEqual(args.trial_table, "tt.h5") + self.assertEqual(args.microscope, "slap2") self.assertEqual(args.scan_mode, "band") self.assertFalse(args.draw_user_rois) self.assertIsNone(args.interactive) @@ -48,12 +63,29 @@ def test_interactive_flag(self): args = self.parser.parse_args(["annotate", "tt.h5", "--interactive"]) self.assertTrue(args.interactive) + def test_supported_routing_choices(self): + """Both implemented commands accept explicit SLAP2 band routing.""" + for command in ("annotate", "extract"): + with self.subTest(command=command): + args = self.parser.parse_args( + [ + command, + "tt.h5", + "--microscope", + "slap2", + "--scan-mode", + "band", + ] + ) + self.assertEqual(args.microscope, "slap2") + self.assertEqual(args.scan_mode, "band") -class TestParamsFromArgs(unittest.TestCase): - """_params_from_args builds a SiloParams from the namespace.""" + +class TestOptionsFromArgs(unittest.TestCase): + """CLI separates science, execution and annotation configuration.""" def test_maps_fields(self): - """Each parsed field maps onto the SiloParams.""" + """Run and annotation flags do not change scientific defaults.""" args = cli.build_parser().parse_args( [ "extract", @@ -67,23 +99,72 @@ def test_maps_fields(self): "--verbose", ] ) - params = cli._params_from_args(args) - self.assertEqual(params.scan_mode, "band") - self.assertTrue(params.draw_user_rois) - self.assertTrue(params.interactive) - self.assertEqual(params.operator, "Alice") - self.assertTrue(params.verbose) + params, execution, annotations = cli._options_from_args(args) + self.assertEqual(params.peakth, 7.0) + self.assertFalse(hasattr(params, "scan_mode")) + self.assertTrue(annotations.enabled) + self.assertTrue(annotations.interactive) + self.assertEqual(annotations.operator, "Alice") + self.assertTrue(execution.verbose) + self.assertEqual(execution.max_workers, 6) + self.assertIsNone(execution.max_trials) + + def test_explicit_annotation_enables_rois(self): + """The annotate command itself enables ROI handling.""" + args = cli.build_parser().parse_args(["annotate", "tt.h5"]) + _, _, annotations = cli._options_from_args(args) + self.assertTrue(annotations.enabled) + + def test_run_limits(self): + """Execution limits map to validated options.""" + args = cli.build_parser().parse_args( + ["extract", "tt.h5", "--max-workers", "2", "--max-trials", "3"] + ) + _, execution, _ = cli._options_from_args(args) + self.assertEqual(execution.max_workers, 2) + self.assertEqual(execution.max_trials, 3) class TestMain(unittest.TestCase): """main dispatches to handlers and handles the no-command case.""" + def test_help_does_not_import_extraction_or_gui_backends(self): + """Fresh-process CLI help exits successfully without heavy imports.""" + code = """ +import sys +from giant_python.cli import main +try: + main(sys.argv[1:]) +except SystemExit as error: + assert error.code == 0, error.code +else: + raise AssertionError('help did not exit') +blocked = ('torch', 'cv2', 'tkinter', 'slap2_utils', + 'giant_python.extraction.band.workflow', + 'giant_python.gui') +loaded = [name for name in sys.modules + if any(name == prefix or name.startswith(prefix + '.') + for prefix in blocked)] +assert not loaded, loaded +""" + for args in (["-h"], ["extract", "-h"], ["annotate", "-h"]): + with self.subTest(args=args): + result = subprocess.run( + [sys.executable, "-c", code, *args], + capture_output=True, + text=True, + timeout=30, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn("usage:", result.stdout) + def test_no_command_prints_help(self): """No subcommand prints help and returns 2.""" parser = cli.build_parser() - with mock.patch.object( - cli, "build_parser", return_value=parser - ), mock.patch.object(parser, "print_help") as help_: + with ( + mock.patch.object(cli, "build_parser", return_value=parser), + mock.patch.object(parser, "print_help") as help_, + ): code = cli.main([]) help_.assert_called_once() self.assertEqual(code, 2) @@ -98,5 +179,118 @@ def test_dispatches_annotate(self): self.assertEqual(code, 0) +class TestCommandServices(unittest.TestCase): + """Handlers delegate to public services and report invalid requests.""" + + def test_extract_service(self): + """The extraction handler passes a filename, not a fake table.""" + with mock.patch( + "giant_python.pipeline.extract.extract_sources" + ) as extract: + code = cli.main( + [ + "extract", + "curated_v2.h5", + "--max-trials", + "4", + "--headless", + "--draw-user-rois", + ] + ) + self.assertEqual(code, 0) + self.assertEqual(extract.call_args.args[0], "curated_v2.h5") + self.assertEqual(extract.call_args.kwargs["scan_mode"], "band") + self.assertEqual(extract.call_args.kwargs["execution"].max_trials, 4) + self.assertTrue(extract.call_args.kwargs["annotations"].enabled) + self.assertFalse(extract.call_args.kwargs["annotations"].interactive) + + def test_annotation_service(self): + """Annotation routes through the same public function as Pipeline.""" + with mock.patch( + "giant_python.pipeline.annotate.annotate_rois" + ) as annotate: + code = cli.main(["annotate", "curated_v2.h5", "--headless"]) + self.assertEqual(code, 0) + self.assertEqual(annotate.call_args.args[0], "curated_v2.h5") + self.assertTrue(annotate.call_args.kwargs["annotations"].enabled) + self.assertFalse(annotate.call_args.kwargs["annotations"].interactive) + + def test_removed_commands_fail_before_dispatch(self): + """Old commands cannot fall through to either implemented handler.""" + with ( + mock.patch.object(cli, "_cmd_annotate") as annotate, + mock.patch.object(cli, "_cmd_extract") as extract, + ): + for argv in ( + ["organize", "/data", "/results"], + ["register", "curated_v2.h5"], + ): + with self.subTest(argv=argv): + with mock.patch( + "sys.stderr", new_callable=io.StringIO + ) as stderr: + with self.assertRaises(SystemExit) as error: + cli.main(argv) + self.assertEqual(error.exception.code, 2) + self.assertIn("invalid choice", stderr.getvalue()) + self.assertIn(argv[0], stderr.getvalue()) + annotate.assert_not_called() + extract.assert_not_called() + + def test_unsupported_routing_reports_parser_error(self): + """Unsupported routing is rejected by argparse before dispatch.""" + with ( + mock.patch.object(cli, "_cmd_annotate") as annotate, + mock.patch.object(cli, "_cmd_extract") as extract, + ): + for command in ("extract", "annotate"): + for flag, mode in ( + ("--scan-mode", "standard"), + ("--scan-mode", "unknown"), + ("--microscope", "bergamo"), + ("--microscope", "unknown"), + ): + with self.subTest(command=command, flag=flag, mode=mode): + with mock.patch( + "sys.stderr", new_callable=io.StringIO + ) as stderr: + with self.assertRaises(SystemExit) as error: + cli.main([command, "tt.h5", flag, mode]) + self.assertEqual(error.exception.code, 2) + self.assertIn("invalid choice", stderr.getvalue()) + self.assertIn(flag, stderr.getvalue()) + self.assertIn(mode, stderr.getvalue()) + annotate.assert_not_called() + extract.assert_not_called() + + def test_invalid_execution_limit_is_reported(self): + """Bad run options fail before the service can load any data.""" + with mock.patch("sys.stderr", new_callable=io.StringIO) as stderr: + with self.assertRaises(SystemExit) as error: + cli.main(["extract", "tt.h5", "--max-workers", "0"]) + self.assertEqual(error.exception.code, 2) + self.assertIn("max_workers", stderr.getvalue()) + + def test_service_value_error_is_actionable(self): + """ValueError from either public service becomes a CLI usage error.""" + services = { + "annotate": "giant_python.pipeline.annotate.annotate_rois", + "extract": "giant_python.pipeline.extract.extract_sources", + } + for command, target in services.items(): + with self.subTest(command=command): + with ( + mock.patch( + target, side_effect=ValueError("Invalid trial table") + ) as service, + mock.patch("sys.stderr", new_callable=io.StringIO) as err, + ): + with self.assertRaises(SystemExit) as error: + cli.main([command, "tt.h5"]) + service.assert_called_once() + self.assertEqual(error.exception.code, 2) + self.assertIn("Invalid trial table", err.getvalue()) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_configuration.py b/tests/test_configuration.py new file mode 100644 index 0000000..bdbd175 --- /dev/null +++ b/tests/test_configuration.py @@ -0,0 +1,184 @@ +"""Contracts for scientific configuration and independent run policies.""" + +import math +import unittest +from copy import deepcopy + +from giant_python.models.params import ( + AnnotationOptions, + BandSiloParams, + ExecutionOptions, + resolve_band_options, +) + + +class TestBandConfiguration(unittest.TestCase): + """Resolution preserves numerical defaults and never mutates inputs.""" + + def test_defaults_match_existing_science(self): + """The split is architectural, not a scientific retuning.""" + science, execution, annotations = resolve_band_options() + self.assertEqual(science.peakth, 7.0) + self.assertEqual(science.denoise_window_s, 1.0) + self.assertEqual(science.vif, 1.38) + self.assertEqual(science.sparse_fac, math.exp(-3.0)) + self.assertEqual(execution, ExecutionOptions(6, False, None)) + self.assertEqual( + annotations, AnnotationOptions(False, None, "SLAP2 User") + ) + self.assertFalse(hasattr(science, "max_workers")) + self.assertFalse(hasattr(science, "scan_mode")) + self.assertFalse(hasattr(science, "interactive")) + + def test_science_rejects_mixed_and_routing_fields(self): + """Science dictionaries never dispatch other scopes or old names.""" + for name, value in ( + ("microscope", "slap2"), + ("scan_mode", "band"), + ("max_workers", 2), + ("max_trials", 3), + ("verbose", True), + ("operator", "Ada"), + ("enabled", True), + ("interactive", False), + ("draw_user_rois", True), + ("sigma_px", 1.5), + ("lambda_", 0.1), + ("phi", None), + ("tau_s", None), + ("photon_scale", None), + ): + with self.subTest(name=name): + with self.assertRaisesRegex(TypeError, name): + resolve_band_options({"peakth": 7.0, name: value}) + + def test_every_result_is_fresh(self): + """Typed inputs and each default result are independent objects.""" + original = (BandSiloParams(), ExecutionOptions(), AnnotationOptions()) + resolved = resolve_band_options(*original) + for actual, expected in zip(resolved, original): + self.assertEqual(actual, expected) + self.assertIsNot(actual, expected) + self.assertIsNot(resolve_band_options()[0], resolved[0]) + + def test_dictionary_inputs_and_overrides_are_not_modified(self): + """Each dictionary affects only its scope and is copied.""" + params = {"analyze_hz": 80.0} + execution = {"max_workers": 2, "verbose": True} + annotations = { + "enabled": True, + "interactive": False, + "operator": "Grace", + } + original = deepcopy((params, execution, annotations)) + science, run, roi = resolve_band_options( + params, execution, annotations + ) + self.assertEqual(science.analyze_hz, 80.0) + self.assertEqual(run, ExecutionOptions(2, True, None)) + self.assertEqual(roi, AnnotationOptions(True, False, "Grace")) + self.assertEqual((params, execution, annotations), original) + + +class TestConfigurationValidation(unittest.TestCase): + """Reject unsupported settings before they reach numerical kernels.""" + + def test_positive_scientific_values(self): + """Rates, windows and positive scale values must be finite.""" + for name in ( + "analyze_hz", + "decay_tau_s", + "baseline_window_s", + "denoise_window_s", + "vif", + "sparse_fac", + "peakth", + ): + for value in (0, -1, math.nan, math.inf, True, "1", None): + with self.subTest(name=name, value=value): + with self.assertRaisesRegex(ValueError, name): + BandSiloParams(**{name: value}) + + def test_positive_counts(self): + """Count overrides require positive integers, not floats or bools.""" + for name in ("d_xy", "peak_buffer", "psf_dilation", "num_channels"): + for value in (0, -1, 1.5, True): + with self.subTest(name=name, value=value): + with self.assertRaisesRegex(ValueError, name): + BandSiloParams(**{name: value}) + self.assertIsNone(BandSiloParams().num_channels) + self.assertEqual(BandSiloParams(num_channels=2).num_channels, 2) + + def test_interpolation_and_activity_channel(self): + """Only existing interpolation modes and activity channel zero work.""" + for mode in ("linear", "cubic"): + self.assertEqual( + BandSiloParams( + background_interpolation=mode + ).background_interpolation, + mode, + ) + with self.assertRaisesRegex(ValueError, "interpolation"): + BandSiloParams(background_interpolation="nearest") + for channel in (1, -1, 0.0, False, None): + with self.subTest(channel=channel): + with self.assertRaisesRegex(ValueError, "activity_channel"): + BandSiloParams(activity_channel=channel) + + def test_execution_and_annotation_policy(self): + """Execution counts and tri-state GUI settings are validated.""" + for name in ("max_workers", "max_trials"): + for value in (0, -1, 1.5, True): + with self.subTest(name=name, value=value): + with self.assertRaisesRegex(ValueError, name): + ExecutionOptions(**{name: value}) + with self.assertRaisesRegex(ValueError, "verbose"): + ExecutionOptions(verbose=1) + for name, value in ( + ("enabled", 1), + ("interactive", "false"), + ("operator", None), + ): + with self.subTest(name=name): + with self.assertRaisesRegex(ValueError, name): + AnnotationOptions(**{name: value}) + + def test_resolver_revalidates_mutated_carriers(self): + """Dataclass mutability cannot bypass validation at resolution.""" + science = BandSiloParams() + science.num_channels = 0 + with self.assertRaisesRegex(ValueError, "num_channels"): + resolve_band_options(science) + execution = ExecutionOptions() + execution.max_workers = 0 + with self.assertRaisesRegex(ValueError, "max_workers"): + resolve_band_options(execution=execution) + annotations = AnnotationOptions() + annotations.interactive = "false" + with self.assertRaisesRegex(ValueError, "interactive"): + resolve_band_options(annotations=annotations) + + def test_bad_input_and_unknown_keys(self): + """Typos and wrong container types do not silently use defaults.""" + for params in (42, {"analyze_hzz": 80}): + with self.assertRaises(TypeError): + resolve_band_options(params) + for keyword, value in ( + ("execution", False), + ("annotations", "headless"), + ("execution", {"max_worker": 2}), + ("annotations", {"draw_user_rois": True}), + ("execution", {"operator": "Ada"}), + ("execution", {"peakth": 7}), + ("annotations", {"max_workers": 2}), + ("annotations", {"scan_mode": "band"}), + ): + with self.subTest(keyword=keyword, value=value): + with self.assertRaises(TypeError): + resolve_band_options(**{keyword: value}) + with self.assertRaises(TypeError): + resolve_band_options({"enabled": True, "draw_user_rois": False}) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_gui.py b/tests/test_gui.py deleted file mode 100644 index ac94e77..0000000 --- a/tests/test_gui.py +++ /dev/null @@ -1,32 +0,0 @@ -"""Tests for the gui subpackage.""" - -import unittest - -from giant_python.gui import DrawROIs, annotate_rois - - -class TestAnnotateRois(unittest.TestCase): - """Tests for the ROI annotation entry point.""" - - def test_annotate_rois(self): - """annotate_rois raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - annotate_rois(None) - - -class TestDrawROIs(unittest.TestCase): - """Tests for the DrawROIs class.""" - - def test_show(self): - """DrawROIs.show raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - DrawROIs(None).show() - - def test_save_rois(self): - """DrawROIs.save_rois raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - DrawROIs(None).save_rois("annotations.h5") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_io.py b/tests/test_io.py index 18d6ad2..931a5ad 100644 --- a/tests/test_io.py +++ b/tests/test_io.py @@ -7,15 +7,9 @@ import h5py import numpy as np -from giant_python.io import ( - get_online_motion, - load_struct_h5, - read_band_trial_data, - ref_pixs_to_drc, - save_struct_h5, - scanimagetiff_data_wrapper, - scanimagetiff_wrapper, -) +from giant_python import io +from giant_python.extraction.band import geometry, trial_data +from giant_python.io import load_struct_h5, slap2 from giant_python.io.hdf5 import _decode_one, _decode_strings @@ -34,7 +28,7 @@ def test_decode_empty_array(self): class TestHdf5(unittest.TestCase): - """Tests for the generic struct <-> HDF5 helpers.""" + """Tests for the generic HDF5 struct reader.""" def test_load_struct_h5_faithful(self): """load_struct_h5 mirrors groups, decodes strings, honors row_major.""" @@ -71,43 +65,29 @@ def test_load_struct_h5_row_major(self): out = load_struct_h5(path) np.testing.assert_array_equal(out["mat"], np.array([[1, 2, 3]])) - def test_save_struct_h5(self): - """save_struct_h5 raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - save_struct_h5({}, "dummy.h5") - - -class TestTiff(unittest.TestCase): - """Tests for the ScanImage TIFF readers.""" - - def test_scanimagetiff_wrapper(self): - """scanimagetiff_wrapper raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - scanimagetiff_wrapper("dummy.tif") - - def test_scanimagetiff_data_wrapper(self): - """scanimagetiff_data_wrapper raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - scanimagetiff_data_wrapper(None, "dummy.tif") - class TestSlap2(unittest.TestCase): - """Tests for SLAP2 readers (online motion + band).""" - - def test_get_online_motion(self): - """get_online_motion raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - get_online_motion(None, np.arange(10)) + """Tests for SLAP2 reader and band reduction ownership.""" def test_ref_pixs_to_drc(self): - """ref_pixs_to_drc raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - ref_pixs_to_drc(np.arange(10), 4, 4) + """Reference-pixel conversion belongs to band geometry, not IO.""" + self.assertTrue(callable(geometry.ref_pixs_to_drc)) + self.assertEqual( + geometry.ref_pixs_to_drc.__module__, geometry.__name__ + ) + for module in (io, slap2): + self.assertFalse(hasattr(module, "ref_pixs_to_drc")) + self.assertNotIn("ref_pixs_to_drc", io.__all__) def test_read_band_trial_data(self): - """read_band_trial_data raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - read_band_trial_data(None, np.arange(5), np.arange(3)) + """Band reduction remains canonical without an IO compatibility API.""" + self.assertTrue(callable(trial_data.read_band_trial_data)) + self.assertEqual( + trial_data.read_band_trial_data.__module__, trial_data.__name__ + ) + for module in (io, slap2): + self.assertFalse(hasattr(module, "read_band_trial_data")) + self.assertNotIn("read_band_trial_data", io.__all__) if __name__ == "__main__": diff --git a/tests/test_io_architecture.py b/tests/test_io_architecture.py new file mode 100644 index 0000000..efb5f1e --- /dev/null +++ b/tests/test_io_architecture.py @@ -0,0 +1,281 @@ +"""IO ownership, faithful decoding, and loaded-TrialTable regression tests.""" + +import ast +import sys +import tempfile +import unittest +from pathlib import Path +from types import ModuleType +from unittest.mock import Mock, patch + +import h5py +import numpy as np + +from giant_python.extraction.band import inputs, result_assembly +from giant_python.io import experiment_summary, hdf5, slap2, trial_table +from giant_python.io.annotations import ( + read_annotations_h5, + save_annotations_h5, +) +from giant_python.io.tiff import read_tiff +from giant_python.io.trial_table import read_trial_table +from giant_python.models.trial_table import Slap2Info, TrialTable + + +class TestSharedDecoder(unittest.TestCase): + """There is one HDF5 decoder and it preserves the storage convention.""" + + def test_removed_aliases_are_unavailable(self): + """There is one public spelling for each raw/schema operation.""" + from giant_python import io + + for owner, names in ( + ( + hdf5, + ( + "load_struct_from_h5", + "_h5_is_string_dataset", + "_decode_h5_strings", + "_read_h5_dataset", + "_read_h5_group", + ), + ), + (slap2, ("read_alignment_data_h5",)), + (trial_table, ("load_trial_table",)), + ( + experiment_summary, + ( + "write_experiment_summary", + "read_experiment_summary", + "path_summary_from_assembled", + "path_summary_to_assembled", + ), + ), + ( + result_assembly, + ("assemble_path_outputs", "build_experiment_summary"), + ), + (io, ("load_struct_from_h5", "write_experiment_summary")), + ): + for name in names: + with self.subTest(owner=owner.__name__, name=name): + self.assertFalse(hasattr(owner, name)) + self.assertIs(inputs.load_struct_h5, hdf5.load_struct_h5) + self.assertIs( + inputs.read_alignment_data_h5, slap2.load_alignment_data_h5 + ) + + def test_layout_flags_and_string_grids(self): + """Missing/zero flags reverse 3D axes; rank-two vectors survive.""" + cube = np.arange(24).reshape(2, 3, 4) + labels = np.array([["a", "b", "c"], ["d", "e", "f"]], dtype=object) + with tempfile.TemporaryDirectory() as dr: + for flag in (None, 0, 1): + with self.subTest(flag=flag): + path = Path(dr) / "layout.h5" + with h5py.File(path, "w") as f: + if flag is not None: + f["row_major"] = flag + f["cube"] = cube + f["vector"] = np.arange(3).reshape(1, 3) + f.create_dataset( + "labels", data=labels, dtype=h5py.string_dtype() + ) + decoded = hdf5.load_struct_h5(path) + np.testing.assert_array_equal( + decoded["cube"], cube if flag else cube.T + ) + np.testing.assert_array_equal( + decoded["labels"], labels if flag else labels.T + ) + self.assertEqual( + decoded["vector"].shape, (1, 3) if flag else (3, 1) + ) + + def test_raw_alignment_sign_and_private_adapter(self): + """Raw displacement stays unchanged; the backend negates offsets.""" + saved = np.array([1.25, -2.5, np.nan]) + with tempfile.TemporaryDirectory() as dr: + path = Path(dr) / "alignment.h5" + with h5py.File(path, "w") as f: + f["row_major"] = 1 + f["DSframes"] = np.array([[1, 5, 9]]) + f["numChannels"] = 2 + f["alignHz"] = 10.0 + for key in ("motionDSr", "motionDSc", "motionDSz"): + f[key] = saved.reshape(-1, 1) + f.create_group("slap2")["onlineMotionXshift"] = saved.reshape( + 1, -1 + ) + raw = slap2.load_alignment_data_h5(path) + with patch.object( + inputs, "read_alignment_data_h5", return_value=raw + ): + normalized = inputs.load_alignment_data_h5(path) + for key in ("motionDSr", "motionDSc", "motionDSz"): + np.testing.assert_array_equal(raw[key], saved) + np.testing.assert_array_equal(normalized[key], -saved) + self.assertFalse(np.shares_memory(raw[key], normalized[key])) + self.assertIs(normalized["onlineXshift"], raw["onlineXshift"]) + self.assertIsNone(normalized["onlineYshift"]) + + +class TestTrialTableBoundary(unittest.TestCase): + """Models retain raw metadata; backend views do path/file preparation.""" + + def test_in_memory_table_is_never_reloaded(self): + """A loaded model keeps identity without reconstructing a filename.""" + table = TrialTable( + datadr=Path("raw"), + savedr=Path("results"), + filename=np.array([["trial.dat"]], dtype=object), + slap2_info=Slap2Info(first_line=[[1]], last_line=[[9]]), + motion_correction={"fn_adata": np.array([["trial.h5"]])}, + source_path=Path("unusual-name.h5"), + ) + with patch.object( + TrialTable, "from_h5", side_effect=AssertionError("reload") + ): + view = inputs.load_trial_table(table) + self.assertIs(view["trial_table"], table) + self.assertEqual(view["source_path"], Path("unusual-name.h5")) + self.assertEqual( + view["fn_adata"][0, 0], + str(Path("results/motion_correction/trial.h5")), + ) + self.assertNotIn("keep_trials", view) + self.assertNotIn("align_hz", view) + + def test_model_codec_records_source_without_probing(self): + """Noncanonical table filenames survive the IO-model delegation.""" + with tempfile.TemporaryDirectory() as dr: + path = Path(dr) / "not-the-standard-name.h5" + with h5py.File(path, "w") as f: + f["row_major"] = 1 + f["datadr"] = "not-a-real-directory" + f.create_dataset( + "filename", data=[["x.dat"]], dtype=h5py.string_dtype() + ) + f["true_trial_ix"] = np.array([[4]]) + f.create_group("slap2_info")["first_line"] = np.array([[1]]) + table = TrialTable.from_h5(path) + self.assertEqual(table.source_path, path.absolute()) + self.assertEqual(table.filename.shape, (1, 1)) + np.testing.assert_array_equal(table.true_trial_ix, [[4]]) + with h5py.File(path, "r") as f: + self.assertNotIn("source_path", f) + self.assertEqual( + read_trial_table(path).source_path, table.source_path + ) + + +class TestRawAnnotations(unittest.TestCase): + """Raw annotations are readable without any reference-stack geometry.""" + + def test_provenance_overlap_and_empty_path(self): + """Masks and provenance survive; empty Path groups are explicit.""" + mask = np.ones((2, 3, 4), dtype=bool) + with tempfile.TemporaryDirectory() as dr: + file = save_annotations_h5( + dr, + {"DMD1": [{"label": "A"}, {"label": "B"}]}, + {"DMD1": [mask, mask]}, + 2, + ref_files={"DMD1": str(Path(dr) / "reference.tif")}, + ) + raw = read_annotations_h5(file) + self.assertEqual(raw["Path1"]["fn"], "reference.tif") + self.assertEqual(int(raw["Path2"]["n_rois"]), 0) + np.testing.assert_array_equal(raw["Path1"]["roi_000"]["mask"], mask) + np.testing.assert_array_equal(raw["Path1"]["roi_001"]["mask"], mask) + + +class TestRawAcquisitionSeams(unittest.TestCase): + """Raw reader construction and TIFF reads preserve native contracts.""" + + def test_open_preserves_vendor_selection_and_caller_ownership(self): + """Opening reads/closes nothing; CYCLE names use MultiDataFiles.""" + vendor = ModuleType("slap2_utils") + single, multiple = Mock(), Mock() + vendor.DataFile = Mock(return_value=single) + vendor.MultiDataFiles = Mock(return_value=multiple) + with patch.dict(sys.modules, {"slap2_utils": vendor}): + with patch("importlib.reload", return_value=vendor) as reload: + self.assertIs(slap2.open_slap2_file("trial.dat"), single) + self.assertIs( + slap2.open_slap2_file("trial_CYCLE001.dat"), multiple + ) + self.assertEqual(reload.call_count, 2) + vendor.DataFile.assert_called_once_with("trial.dat") + vendor.MultiDataFiles.assert_called_once_with("trial_CYCLE001.dat") + single.getLineData.assert_not_called() + multiple.getLineData.assert_not_called() + single.close.assert_not_called() + multiple.close.assert_not_called() + + def test_native_tiff_shape_dtype_and_intensity(self): + """Shared TIFF decoding does not perform band reference scaling.""" + import tifffile + + array = np.arange(60, dtype=np.uint16).reshape(3, 4, 5) + with tempfile.TemporaryDirectory() as dr: + path = Path(dr) / "reference.tif" + tifffile.imwrite(path, array, photometric="minisblack") + loaded = read_tiff(path) + self.assertEqual(loaded.dtype, array.dtype) + np.testing.assert_array_equal(loaded, array) + + +class TestImportOwnership(unittest.TestCase): + """Source-level import boundaries remain independent of heavy backends.""" + + def test_io_and_models_do_not_import_band_or_toolkits(self): + """Shared readers and models do not import band/numerics/GUI.""" + root = Path(__file__).resolve().parents[1] / "src/giant_python" + paths = list((root / "io").glob("*.py")) + paths += [ + root / "models/experiment_summary.py", + root / "models/trial_table.py", + ] + for path in paths: + with self.subTest(path=path.name): + tree = ast.parse(path.read_text(encoding="utf-8")) + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + module = node.module or "" + self.assertFalse( + any( + part in module.split(".") + for part in ( + "bandsilo", + "extraction", + "torch", + "cv2", + "tkinter", + ) + ) + ) + + def test_numerical_assembly_constructs_models_without_io(self): + """Results may depend on models/numerics, never persistence or GUI.""" + tree = ast.parse( + Path(result_assembly.__file__).read_text(encoding="utf-8") + ) + forbidden = {"io", "h5py", "workflow", "gui", "bandsilo"} + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + self.assertFalse( + forbidden.intersection((node.module or "").split(".")) + ) + elif isinstance(node, ast.Import): + for alias in node.names: + self.assertFalse( + forbidden.intersection(alias.name.split(".")) + ) + public_functions = { + node.name + for node in tree.body + if isinstance(node, ast.FunctionDef) + and not node.name.startswith("_") + } + self.assertEqual(public_functions, {"assemble_path_summary"}) diff --git a/tests/test_io_edges.py b/tests/test_io_edges.py new file mode 100644 index 0000000..4e3ef28 --- /dev/null +++ b/tests/test_io_edges.py @@ -0,0 +1,523 @@ +"""Regression coverage for schema validation and public IO/model seams.""" + +import importlib +import sys +import unittest +from dataclasses import fields, replace +from pathlib import Path +from tempfile import TemporaryDirectory +from unittest.mock import patch + +import h5py +import numpy as np + +from giant_python.extraction.band import annotation, inputs +from giant_python.io.experiment_summary import read_summary, write_summary +from giant_python.io.hdf5 import write_dict_to_h5group +from giant_python.models.experiment_summary import ( + FRAME_INFO_KEYS, + ExperimentSummary, + FrameInfo, + PathSummary, + Source, + UserRoi, + Visualizations, +) +from giant_python.models.params import ( + AnnotationOptions, + BandSiloParams, + ExecutionOptions, +) +from giant_python.models.trial_table import Slap2Info + + +def _empty_source_path(**kwargs): + """Provide empty band tensors with explicit rank/dtype templates.""" + values = { + "z_depths": np.array([[1], [2]], dtype=np.int32), + "global_f": np.ones((2, 5), dtype=np.float32), + "frame_info": FrameInfo( + **{key: np.zeros((5, 1)) for key in FRAME_INFO_KEYS} + ), + "visualizations": Visualizations( + mean_im=np.ones((2, 2, 3, 4), dtype=np.float32), + act_im=np.ones((2, 3, 4), dtype=np.float32), + act_im_peaks=np.empty((0, 3), dtype=np.int32), + ), + **kwargs, + } + return PathSummary( + _empty_source_arrays={ + "profiles": np.empty((0, 2, 3, 4), dtype=np.float32), + "coords": np.empty((0, 3), dtype=np.int32), + "dF_ls": np.empty((0, 2, 5), dtype=np.float32), + "F0": np.empty((0, 2, 5), dtype=np.float32), + "SNR": np.empty((0, 1), dtype=np.float32), + }, + **values, + ) + + +def _roundtrip(path): + """Exercise public model persistence in a real temporary HDF5 file.""" + with TemporaryDirectory() as directory: + destination = Path(directory) / "summary.h5" + write_summary(ExperimentSummary(paths=[path]), destination) + return read_summary(destination).paths[0] + + +class TestSummaryValidation(unittest.TestCase): + """Invalid models fail explicitly rather than silently losing ROI data.""" + + def test_missing_arrays_preserve_existing_file_groups(self): + """Report missing arrays by field/index before replacing the path.""" + source = Source( + profile=np.ones((1, 2, 3), dtype=np.float32), + coords=np.array([0, 1, 2], dtype=np.int32), + df_ls=np.ones((1, 4), dtype=np.float32), + f0=np.ones((1, 4), dtype=np.float32), + snr=2.0, + ) + roi = UserRoi( + label="retained μ", + mask=np.ones((1, 2, 3), dtype=bool), + f=np.ones((1, 4), dtype=np.float32), + ) + shifts = np.array([[1.5, -2.0, np.nan]], dtype=np.float32) + with TemporaryDirectory() as directory: + destination = Path(directory) / "summary.h5" + with h5py.File(destination, "w") as file: + file["row_major"] = 1 + file.create_group("params").attrs["owner"] = "original" + for name in ("Path1", "Path2"): + group = file.create_group(name) + group.attrs["marker"] = name + group.create_group("frame_info").create_dataset( + "offlineXshifts", data=shifts + ) + for owner, field in ( + ("Source", "profile"), + ("Source", "coords"), + ("Source", "df_ls"), + ("Source", "f0"), + ("UserRoi", "mask"), + ("UserRoi", "f"), + ): + with self.subTest(owner=owner, field=field): + path = PathSummary( + path_index=1, + sources=[source, replace(source)], + user_rois=[roi, replace(roi)], + annotation_enabled=True, + ) + entry = ( + path.sources[1] + if owner == "Source" + else path.user_rois[1] + ) + setattr(entry, field, None) + message = rf"{owner}\.{field}\[1\] is required; got None" + with self.assertRaisesRegex(ValueError, message): + write_summary( + ExperimentSummary(paths=[path]), destination + ) + with h5py.File(destination, "r") as file: + self.assertEqual( + set(file), + {"row_major", "params", "Path1", "Path2"}, + ) + self.assertEqual(file["row_major"][()], 1) + self.assertEqual( + file["params"].attrs["owner"], "original" + ) + self.assertEqual(set(file["params"]), set()) + for name in ("Path1", "Path2"): + group = file[name] + self.assertEqual(group.attrs["marker"], name) + self.assertEqual(set(group), {"frame_info"}) + self.assertEqual( + set(group["frame_info"]), {"offlineXshifts"} + ) + saved = group["frame_info/offlineXshifts"] + np.testing.assert_array_equal(saved[()], shifts) + self.assertEqual(saved.dtype, shifts.dtype) + + def test_sources_without_shape_templates_are_rejected(self): + """Empty source lists need geometry that cannot be inferred.""" + with self.assertRaisesRegex(ValueError, "shape templates"): + _roundtrip(PathSummary()) + + def test_disabled_annotations_reject_supplied_rois(self): + """Callers must explicitly enable serialization of supplied ROIs.""" + path = _empty_source_path(user_rois=[UserRoi(label="keep me")]) + with self.assertRaisesRegex(ValueError, "annotation_enabled=True"): + _roundtrip(path) + self.assertEqual(path.user_rois[0].label, "keep me") + self.assertFalse(path.annotation_enabled) + + def test_direct_empty_rois_infer_layout_and_preserve_dtype(self): + """Enabled empty ROIs infer mask/trace geometry without mutation.""" + mean = np.ones((2, 2, 3, 4), dtype=np.float32) + global_f = np.ones((2, 5), dtype=np.float64) + path = _empty_source_path( + annotation_enabled=True, + visualizations=Visualizations( + mean_im=mean, + act_im=np.ones((2, 3, 4)), + act_im_peaks=np.empty((0, 3)), + ), + global_f=global_f, + ) + out = _roundtrip(path) + rois = out._empty_roi_arrays + self.assertEqual(rois["mask"].shape, (0, 2, 3, 4)) + self.assertEqual(rois["mask"].dtype, np.dtype(bool)) + self.assertEqual(rois["F"].shape, (0, 2, 5)) + self.assertEqual(rois["F"].dtype, global_f.dtype) + self.assertEqual(out.user_rois, []) + self.assertIs(path.visualizations.mean_im, mean) + self.assertIs(path.global_f, global_f) + np.testing.assert_array_equal(out.visualizations.mean_im, mean) + np.testing.assert_array_equal(out.global_f, global_f) + self.assertEqual(path._empty_roi_arrays, {}) + + def test_empty_rois_require_both_geometry_ranks(self): + """Neither malformed image geometry nor trace geometry is guessed.""" + for mean_shape, trace_shape in ( + ((2, 3, 4), (2, 5)), + ((2, 2, 3, 4), (5,)), + ): + with self.subTest(mean=mean_shape, traces=trace_shape): + path = _empty_source_path( + annotation_enabled=True, + visualizations=Visualizations( + mean_im=np.zeros(mean_shape) + ), + global_f=np.zeros(trace_shape), + ) + with self.assertRaisesRegex(ValueError, "geometry"): + _roundtrip(path) + + def test_duplicate_path_ids_fail_before_opening_destination(self): + """Duplicate identities cannot replace data in a destination file.""" + summary = ExperimentSummary( + paths=[PathSummary(path_index=3), PathSummary(path_index=3)] + ) + with patch("giant_python.io.experiment_summary.h5py.File") as opener: + with self.assertRaisesRegex(ValueError, "duplicate path_index"): + write_summary(summary, "must-not-open.h5") + opener.assert_not_called() + + def test_model_identity_validation_and_empty_fields(self): + """Path IDs are nonnegative integers; empty summaries stay empty.""" + for value in (-1, 1.5, "1", None): + with self.subTest(value=value): + with self.assertRaisesRegex(ValueError, "nonnegative"): + PathSummary(path_index=value) + self.assertEqual(PathSummary(path_index=np.int64(2)).path_index, 2) + summary = ExperimentSummary() + self.assertEqual(summary.paths, []) + self.assertIsNone(summary.params) + self.assertEqual( + {item.name for item in fields(summary)}, {"paths", "params"} + ) + info = FrameInfo() + self.assertEqual(tuple(info.to_dict()), FRAME_INFO_KEYS) + self.assertTrue( + all(value is None for value in info.to_dict().values()) + ) + + def test_path_fields_share_original_arrays(self): + """Explicit path access preserves array identity, dtype and rank.""" + path = PathSummary( + global_f=np.ones((2, 5), dtype=np.float32), + z_depths=np.array([[1], [3]], dtype=np.int16), + ) + summary = ExperimentSummary(paths=[path]) + self.assertIs(summary.paths[0].global_f, path.global_f) + self.assertIs(summary.paths[0].z_depths, path.z_depths) + self.assertIs(summary.paths[0].visualizations, path.visualizations) + + def test_append_retains_layout_flags_and_unrelated_groups(self): + """Replacement preserves root metadata, including MATLAB flags.""" + for layout in (0, 1): + with ( + self.subTest(layout=layout), + TemporaryDirectory() as directory, + ): + destination = Path(directory) / "summary.h5" + with h5py.File(destination, "w") as file: + file["row_major"] = np.uint8(layout) + file["coords_zero_indexed"] = np.uint8(0) + file["params/operator"] = "original μ" + file["Path1/obsolete"] = 5 + file["Path12/retained"] = np.array([[2, 7]], np.int16) + file["unrelated/labels"] = "retained μ" + path = _empty_source_path() + write_summary( + ExperimentSummary( + paths=[path], params={"operator": "replacement"} + ), + destination, + ) + with h5py.File(destination, "r") as file: + self.assertEqual(file["row_major"][()], layout) + self.assertEqual(file["row_major"].dtype, np.uint8) + self.assertEqual(file["coords_zero_indexed"][()], 0) + self.assertEqual( + file["params/operator"].asstr()[()], "original μ" + ) + self.assertEqual( + file["unrelated/labels"].asstr()[()], "retained μ" + ) + self.assertNotIn("obsolete", file["Path1"]) + np.testing.assert_array_equal( + file["Path1/Z_depths"], path.z_depths + ) + np.testing.assert_array_equal( + file["Path12/retained"], [[2, 7]] + ) + self.assertEqual(file["Path12/retained"].dtype, np.int16) + + +class TestBandInputValidation(unittest.TestCase): + """Missing lookup/reference groups fail at the DMD input boundary.""" + + def test_missing_lookup_groups_identify_requested_dmd(self): + """Absent or None aliases fail after any preceding valid DMD group.""" + valid = { + "allSuperPixelIDs": np.array([1]), + "sparseMaskInds": np.array([[1, 2]]), + "fastZ2RefZ": np.array([1]), + } + for missing in ({}, {"Path2": None}, {"DMD2": None}): + with self.subTest(missing=missing): + raw = {"Path1": valid, **missing} + with patch.object(inputs, "load_struct_h5", return_value=raw): + with self.assertRaisesRegex( + ValueError, "Missing lookup-table group Path2 or DMD2" + ): + inputs.load_lookup_table("lookup.h5", 2) + + def test_missing_reference_groups_fail_before_file_discovery(self): + """Distinguish missing metadata from a missing reference TIFF.""" + for missing in ({}, {"Path2": None}, {"DMD2": None}): + with self.subTest(missing=missing): + raw = {"Path1": {"channels": np.array([1])}, **missing} + with ( + patch.object(inputs, "find_reference_file") as find, + patch.object(inputs, "read_tiff") as read, + ): + with self.assertRaisesRegex( + ValueError, + "Missing reference-stack group Path2 or DMD2", + ): + inputs.load_reference_stack("raw", raw, 1) + find.assert_not_called() + read.assert_not_called() + + +class TestParameterWriterEdges(unittest.TestCase): + """Arrays and Python sequences retain their persisted string policies.""" + + def test_workflow_parameters_keep_numeric_storage(self): + """Canonical Python params retain scalar types and UTF-8 labels.""" + summary = ExperimentSummary( + params={ + "numChannels": 2, + "analyzeHz": 100.0, + "draw_user_rois": True, + "operator": "operator μ", + } + ) + with TemporaryDirectory() as directory: + destination = Path(directory) / "summary.h5" + write_summary(summary, destination) + with h5py.File(destination, "r") as file: + expected = { + "numChannels": (2, np.int64), + "analyzeHz": (100.0, np.float64), + "draw_user_rois": (1, np.uint8), + } + for name, (value, dtype) in expected.items(): + dataset = file[f"params/{name}"] + self.assertEqual(dataset.shape, ()) + self.assertEqual(dataset.dtype, dtype) + self.assertEqual(dataset[()], value) + self.assertEqual( + file["params/operator"].asstr()[()], "operator μ" + ) + + def test_preserved_strings_and_sequences(self): + """UTF-8 grids retain shape; sequence bytes keep their str spelling.""" + values = { + "unicode": np.array([["μ", ""], ["é", "last"]]), + "bytes": np.array([[b"first", b"\xff"]]), + "objects": np.array([["μ", b"tail"]], dtype=object), + "scalar": np.str_("μ"), + "empty": np.empty((0, 2), dtype="U1"), + "nested": {"flag": np.uint8(1)}, + } + with h5py.File( + "parameter-edges.h5", "w", driver="core", backing_store=False + ) as file: + write_dict_to_h5group(file, values) + np.testing.assert_array_equal( + file["unicode"].asstr()[()], values["unicode"] + ) + np.testing.assert_array_equal( + file["bytes"].asstr()[()], [["first", "�"]] + ) + np.testing.assert_array_equal( + file["objects"].asstr()[()], [["μ", "tail"]] + ) + self.assertEqual(file["scalar"].asstr()[()], "μ") + self.assertEqual(file["empty"].shape, (0, 2)) + self.assertEqual(file["nested/flag"].dtype, np.dtype("uint8")) + legacy = file.create_group("legacy") + write_dict_to_h5group( + legacy, + { + "bytes": [b"first", b"last"], + "numbers": (1, 2), + "empty": [], + "none": None, + }, + ) + np.testing.assert_array_equal( + legacy["bytes"].asstr()[()], ["b'first'", "b'last'"] + ) + np.testing.assert_array_equal(legacy["numbers"][()], [1, 2]) + self.assertNotIn("empty", legacy) + self.assertNotIn("none", legacy) + + +class TestModelAndImportBoundaries(unittest.TestCase): + """Canonical model readers and public imports keep their ownership.""" + + def test_trial_table_metadata_adapter(self): + """Raw metadata decoding filters unknown fields without copying.""" + from giant_python.io.trial_table import slap2_info_from_raw + + first_line = np.array([[1, 9]], dtype=np.int32) + raw = {"first_line": first_line, "unknown": "ignored"} + info = slap2_info_from_raw(raw) + self.assertIsInstance(info, Slap2Info) + self.assertIs(info.first_line, first_line) + self.assertFalse(hasattr(info, "unknown")) + self.assertIsNone(slap2_info_from_raw(None)) + self.assertIsNone(slap2_info_from_raw({})) + + def test_lazy_io_exports_and_unknown_attribute(self): + """Schema exports resolve and cache canonical functions lazily.""" + from giant_python import io + + importlib.reload(io) + for name, owner in io._SCHEMA_EXPORTS.items(): + with self.subTest(name=name): + expected = getattr( + importlib.import_module(owner, io.__name__), name + ) + self.assertIs(io.__getattr__(name), expected) + self.assertIs(getattr(io, name), expected) + self.assertIn(name, io.__all__) + with self.assertRaisesRegex(AttributeError, "unknown_codec"): + io.__getattr__("unknown_codec") + + def test_annotation_and_options_import_without_toolkits(self): + """Headless orchestration and scoped option metadata need no GUI.""" + with patch.dict(sys.modules, {"tkinter": None, "cv2": None}): + importlib.reload(annotation) + self.assertTrue(callable(annotation.annotate_band_rois)) + science = {item.name for item in fields(BandSiloParams)} + execution = {item.name for item in fields(ExecutionOptions)} + annotations = {item.name for item in fields(AnnotationOptions)} + self.assertFalse(science & execution) + self.assertFalse(science & annotations) + self.assertFalse(execution & annotations) + + +class TestAnnotationGeometryAssembly(unittest.TestCase): + """Per-DMD geometry uses each path's first kept trial and provenance.""" + + def test_geometry_uses_first_kept_trial_for_each_dmd(self): + """Distinct motion offsets and references stay associated by DMD.""" + references = [np.ones((1, 2, 5, 6)), np.full((1, 2, 5, 6), 2.0)] + table = { + "datadr": Path("raw"), + "ref_stack": {"metadata": "retained"}, + "n_dmds": 2, + "keep_trials": np.array([[False, True], [True, False]]), + "fn_adata": np.array([["skip-a", "a.h5"], ["b.h5", "skip-b"]]), + } + lookup = { + "allSuperPixelIDs": {"DMD1": [11], "DMD2": [22]}, + "sparseMaskInds": {"DMD1": [33], "DMD2": [44]}, + "fastZ2RefZ": {"DMD1": np.array([1, 2]), "DMD2": np.array([1, 2])}, + } + motion = [ + {"motionDSr": [1.0], "motionDSc": [-1.0], "motionDSz": [0.0]}, + {"motionDSr": [-1.0], "motionDSc": [1.0], "motionDSz": [0.0]}, + ] + with ( + patch.object( + annotation.inputs, + "load_reference_stack", + side_effect=[ + (references[0], None, "ref-a.tif"), + (references[1], None, "ref-b.tif"), + ], + ) as load_reference, + patch.object( + annotation.geo, + "build_subsample_matrix_inds", + return_value=np.array([[6], [36]]), + ) as subsample, + patch.object( + annotation, "load_alignment_data_h5", side_effect=motion + ) as load_alignment, + ): + geometry, provenance = annotation.build_user_roi_geometry( + table, lookup + ) + self.assertEqual( + provenance, {"DMD1": "ref-a.tif", "DMD2": "ref-b.tif"} + ) + self.assertEqual( + [call.args[0] for call in load_alignment.call_args_list], + ["a.h5", "b.h5"], + ) + for index, key in enumerate(("DMD1", "DMD2")): + with self.subTest(path=key): + self.assertIs(geometry[key]["ref"], references[index]) + np.testing.assert_array_equal(geometry[key]["z_map"], [0, 1]) + np.testing.assert_array_equal( + geometry[key]["sp_rows"], [2, 2] if index == 0 else [0, 0] + ) + np.testing.assert_array_equal( + geometry[key]["sp_cols"], [0, 0] if index == 0 else [2, 2] + ) + self.assertEqual(np.count_nonzero(geometry[key]["sp_mask"]), 2) + self.assertEqual( + load_reference.call_args_list[index].args, + ("raw", table["ref_stack"], index), + ) + self.assertEqual( + subsample.call_args_list[index].args, + ( + lookup["allSuperPixelIDs"][key], + lookup["sparseMaskInds"][key], + ), + ) + np.testing.assert_array_equal( + table["keep_trials"], [[False, True], [True, False]] + ) + + def test_annotation_option_resolution_preserves_input(self): + """Annotation uses scoped option conversion without mutation.""" + original = AnnotationOptions(operator="operator μ", interactive=False) + _, _, resolved = annotation.resolve_band_options(annotations=original) + self.assertEqual(resolved.operator, "operator μ") + self.assertFalse(resolved.interactive) + self.assertEqual(original.operator, "operator μ") + self.assertFalse(original.interactive) diff --git a/tests/test_math.py b/tests/test_math.py deleted file mode 100644 index 67762ac..0000000 --- a/tests/test_math.py +++ /dev/null @@ -1,110 +0,0 @@ -"""Tests for the math subpackage.""" - -import unittest - -import numpy as np - -from giant_python.math import ( - compute_f0, - detect_peaks_2d, - dft_register_clipped, - gaussian_peaks_integrated, - get_act_im_peaks, - interp_frame, - xcorr2_nans, - xcorr2_nans3d, - xcorr2_nans_weighted, -) -from giant_python.math.deconv import deconvolve_trace -from giant_python.math.nmf import nmf_decompose -from giant_python.math.variance import activity_image - - -class TestRegistration(unittest.TestCase): - """Tests for registration kernels.""" - - def test_xcorr2_nans_weighted(self): - """xcorr2_nans_weighted raises NotImplementedError.""" - img = np.zeros((4, 4)) - with self.assertRaises(NotImplementedError): - xcorr2_nans_weighted(img, img, img, np.zeros(2), 3.0) - - def test_xcorr2_nans(self): - """xcorr2_nans raises NotImplementedError.""" - img = np.zeros((4, 4)) - with self.assertRaises(NotImplementedError): - xcorr2_nans(img, img) - - def test_xcorr2_nans3d(self): - """xcorr2_nans3d raises NotImplementedError.""" - vol = np.zeros((2, 4, 4)) - with self.assertRaises(NotImplementedError): - xcorr2_nans3d(vol, vol) - - def test_dft_register_clipped(self): - """dft_register_clipped raises NotImplementedError.""" - img = np.zeros((4, 4)) - with self.assertRaises(NotImplementedError): - dft_register_clipped(img, img) - - -class TestInterpolation(unittest.TestCase): - """Tests for interp_frame.""" - - def test_interp_frame(self): - """interp_frame raises NotImplementedError.""" - img = np.zeros((4, 4)) - with self.assertRaises(NotImplementedError): - interp_frame(img, img, img, img) - - -class TestExtractionKernels(unittest.TestCase): - """Tests for nmf/deconv/variance kernels.""" - - def test_nmf_decompose(self): - """nmf_decompose raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - nmf_decompose(np.zeros((16, 10)), 2) - - def test_deconvolve_trace(self): - """deconvolve_trace raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - deconvolve_trace(np.zeros(10), 0.5, 30.0) - - def test_activity_image(self): - """activity_image raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - activity_image(np.zeros((10, 4, 4)), 1.0) - - -class TestPeaks(unittest.TestCase): - """Tests for the shared peak-detection kernels.""" - - def test_gaussian_peaks_integrated(self): - """gaussian_peaks_integrated raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - gaussian_peaks_integrated(np.zeros((1, 4)), np.zeros((4, 2))) - - def test_detect_peaks_2d(self): - """detect_peaks_2d raises NotImplementedError.""" - img = np.zeros((8, 8)) - with self.assertRaises(NotImplementedError): - detect_peaks_2d(img, np.zeros((8, 8), bool), 0.0, 1.0, 3.0, 3.0) - - def test_get_act_im_peaks(self): - """get_act_im_peaks raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - get_act_im_peaks(np.zeros((2, 8, 8))) - - -class TestBaseline(unittest.TestCase): - """Tests for the shared baseline kernel.""" - - def test_compute_f0(self): - """compute_f0 raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - compute_f0(np.zeros((20, 3)), 5, 8) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_models.py b/tests/test_models.py index d0265a7..e4d4f28 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -3,21 +3,22 @@ import os import tempfile import unittest +from unittest import mock import h5py import numpy as np from giant_python.models import ( - AlignmentData, - AlignParams, + AnnotationOptions, + BandSiloParams, ExperimentSummary, - SiloParams, + FrameInfo, + PathSummary, Slap2Info, Source, TrialTable, UserRoi, Visualizations, - set_params, ) @@ -34,15 +35,25 @@ def test_slap2_info_defaults(self): """Slap2Info constructs with empty collections.""" self.assertEqual(Slap2Info().ref_stack, {}) - def test_alignment_data_defaults(self): - """AlignmentData defaults to a non-failed registration.""" - self.assertFalse(AlignmentData().registration_failed) - def test_experiment_summary_defaults(self): - """ExperimentSummary nests a Visualizations instance.""" + """ExperimentSummary owns per-path results, not aggregate aliases.""" summary = ExperimentSummary() - self.assertIsInstance(summary.visualizations, Visualizations) - self.assertEqual(summary.sources, []) + self.assertEqual(summary.paths, []) + path = PathSummary() + summary.paths.append(path) + self.assertIsInstance(summary.paths[0].visualizations, Visualizations) + self.assertIsInstance(summary.paths[0].frame_info, FrameInfo) + self.assertEqual(summary.paths[0].sources, []) + for name in ( + "sources", + "user_rois", + "visualizations", + "frame_info", + "global_f", + "z_depths", + ): + with self.subTest(name=name): + self.assertFalse(hasattr(summary, name)) def test_source_and_user_roi(self): """Source and UserRoi construct with defaults.""" @@ -51,23 +62,21 @@ def test_source_and_user_roi(self): def test_param_models(self): """Param models expose typed defaults.""" - self.assertEqual(AlignParams().align_hz, 80.0) - self.assertEqual(SiloParams().microscope, "slap2") - self.assertEqual(SiloParams().scan_mode, "standard") - self.assertEqual(SiloParams().background_interpolation, "cubic") + self.assertEqual(BandSiloParams().background_interpolation, "cubic") - def test_silo_value_params_have_concrete_defaults(self): + def test_band_value_params_have_concrete_defaults(self): """Value params default to concrete (non-None) values.""" - p = SiloParams() + p = BandSiloParams() self.assertEqual(p.analyze_hz, 100.0) - self.assertEqual(p.peakth, 8.0) + # Seven is the current implementation default, not a new retuning. + self.assertEqual(p.peakth, 7.0) # Only genuinely runtime-resolved fields stay None sentinels. self.assertIsNone(p.num_channels) - self.assertIsNone(p.interactive) + self.assertIsNone(AnnotationOptions().interactive) class TestH5RoundTrip(unittest.TestCase): - """from_h5/to_h5 are not implemented yet.""" + """Implemented model codecs preserve data and delegate to shared IO.""" def test_trial_table_from_h5(self): """from_h5 faithfully mirrors the trial_table.h5 group hierarchy.""" @@ -125,39 +134,25 @@ def test_trial_table_from_h5_non_slap2(self): self.assertIsNone(tt.slap2_info) self.assertIsNone(tt.motion_correction) - def test_trial_table_to_h5(self): - """TrialTable.to_h5 raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - TrialTable().to_h5("trial_table.h5") - - def test_alignment_data_from_h5(self): - """AlignmentData.from_h5 raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - AlignmentData.from_h5("a.h5") - - def test_alignment_data_to_h5(self): - """AlignmentData.to_h5 raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - AlignmentData().to_h5("a.h5") - def test_experiment_summary_from_h5(self): - """ExperimentSummary.from_h5 raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - ExperimentSummary.from_h5("experiment_summary.h5") + """ExperimentSummary.from_h5 delegates to the shared band codec.""" + expected = ExperimentSummary() + with mock.patch( + "giant_python.io.experiment_summary.read_summary", + return_value=expected, + ) as reader: + result = ExperimentSummary.from_h5("custom_summary.h5") + self.assertIs(result, expected) + reader.assert_called_once_with("custom_summary.h5") def test_experiment_summary_to_h5(self): - """ExperimentSummary.to_h5 raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - ExperimentSummary().to_h5("experiment_summary.h5") - - -class TestSetParams(unittest.TestCase): - """Tests for the set_params compatibility shim.""" - - def test_not_implemented(self): - """set_params raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - set_params("SILo") + """ExperimentSummary.to_h5 delegates without rewriting the model.""" + summary = ExperimentSummary() + with mock.patch( + "giant_python.io.experiment_summary.write_summary" + ) as writer: + self.assertIsNone(summary.to_h5("custom_summary.h5")) + writer.assert_called_once_with(summary, "custom_summary.h5") if __name__ == "__main__": diff --git a/tests/test_motion_convention.py b/tests/test_motion_convention.py new file mode 100644 index 0000000..c7c3e5a --- /dev/null +++ b/tests/test_motion_convention.py @@ -0,0 +1,89 @@ +"""Regression tests for MATLAB-compatible public offline displacement.""" + +import tempfile +import unittest +from pathlib import Path + +import h5py +import numpy as np + +from giant_python.extraction.band.inputs import load_alignment_data_h5 +from giant_python.extraction.band.result_assembly import assemble_path_summary +from giant_python.extraction.band.traces import _interp_trial_alignment +from giant_python.io.experiment_summary import write_summary +from giant_python.models.experiment_summary import ExperimentSummary +from tests.test_bandsilo_pipeline import _path_result + + +class TestMotionConvention(unittest.TestCase): + """Convert private reference offsets back to public displacement once.""" + + def test_alignment_to_summary_preserves_saved_displacement(self): + """Signed fractional motion survives loading, interpolation and IO.""" + with tempfile.TemporaryDirectory() as directory: + alignment = Path(directory) / "alignment.h5" + ds = np.array([1.0, 3.0, 5.0, 7.0]) + frames = np.arange(1.0, 8.0) + displacement = ( + np.array([1.5, -2.0, 0.0, 3.0]), + np.array([-3.0, 2.5, 1.0, -1.0]), + np.array([0.5, -0.5, 1.5, 0.0]), + ) + with h5py.File(alignment, "w") as f: + f["row_major"] = 1 + f["DSframes"] = ds + f["numChannels"] = 1 + f["alignHz"] = 10.0 + for axis, values in zip("rcz", displacement): + f[f"motionDS{axis}"] = values + for axis, value in zip("YXZ", (2, -3, 1)): + f[f"slap2/onlineMotion{axis}shift"] = np.full(4, value) + loaded = load_alignment_data_h5(alignment) + motion = _interp_trial_alignment( + frames, loaded, np.zeros((1, ds.size)) + ) + pr = _path_result() + trial = pr.trace_results[1]._replace( + frame_line_idxs=frames, + reference_offsets=motion[:3], + online_shifts=motion[4:], + ) + pr.trace_results = [trial] + original = [a.copy() for a in trial.reference_offsets] + path = assemble_path_summary(pr) + summary = ExperimentSummary(paths=[path]) + output = Path(directory) / "summary.h5" + write_summary(summary, output) + with h5py.File(output, "r") as f: + for axis, values in zip("YXZ", displacement): + key = f"offline{axis}shifts" + expected = np.interp(frames, ds, values)[:, None] + np.testing.assert_allclose( + getattr(summary.paths[0].frame_info, key), expected + ) + np.testing.assert_allclose( + f[f"Path1/frame_info/{key}"], expected + ) + for axis, value in zip("YXZ", (2, -3, 1)): + key = f"online{axis}shifts" + np.testing.assert_array_equal( + getattr(path.frame_info, key), np.full((7, 1), value) + ) + for before, after in zip(original, trial.reference_offsets): + np.testing.assert_array_equal(before, after) + + def test_output_negation_preserves_nan_and_input_arrays(self): + """NaNs and trial order survive without modifying numerical inputs.""" + pr = _path_result() + pr.trace_results[0].reference_offsets[0][0] = np.nan + original = np.concatenate( + [r.reference_offsets[0] for r in pr.trace_results] + ) + path = assemble_path_summary(pr) + np.testing.assert_array_equal( + path.frame_info.offlineYshifts.ravel(), -original + ) + np.testing.assert_array_equal( + np.concatenate([r.reference_offsets[0] for r in pr.trace_results]), + original, + ) diff --git a/tests/test_no_placeholders.py b/tests/test_no_placeholders.py new file mode 100644 index 0000000..73da82f --- /dev/null +++ b/tests/test_no_placeholders.py @@ -0,0 +1,335 @@ +"""Prevent removed placeholder modules, APIs, and empty functions returning.""" + +import ast +import inspect +import unittest +from importlib import import_module +from importlib.util import find_spec +from pathlib import Path +from textwrap import dedent + +from giant_python.models import TrialTable +from giant_python.pipeline import Pipeline + +PACKAGE_ROOT = Path(__file__).resolve().parents[1] / "src" / "giant_python" + +REMOVED_MODULES = ( + "numerics.deconv", + "numerics.nmf", + "numerics.registration", + "numerics.variance", + "pipeline.base", + "pipeline.organize", + "pipeline.register", + "gui.draw_rois", + "models.alignment", +) +REMOVED_STAGES = ( + "Stage", + "TrialTableBuilder", + "MotionCorrector", + "MultiRoiRegistration", + "StripRegistration", + "BandRegistration", + "build_trial_table", + "build_trial_table_slap2", + "verify_files", + "multi_roi_registration", + "strip_registration", + "band_registration", + "silo", +) +REMOVED_KERNELS = ( + "deconvolve_trace", + "nmf_decompose", + "xcorr2_nans_weighted", + "xcorr2_nans", + "xcorr2_nans3d", + "dft_register_clipped", + "activity_image", + "interp_frame", +) +REMOVED_IO = ( + "save_struct_h5", + "get_online_motion", + "scanimagetiff_wrapper", + "scanimagetiff_data_wrapper", +) +REMOVED_EXPORTS = { + "giant_python": ( + *REMOVED_STAGES, + *REMOVED_KERNELS, + *REMOVED_IO, + "AlignParams", + "AlignmentData", + "DrawROIs", + ), + "giant_python.models": ("AlignParams", "AlignmentData"), + "giant_python.models.params": ("AlignParams",), + "giant_python.pipeline": REMOVED_STAGES, + "giant_python.pipeline.pipeline": (*REMOVED_STAGES, "AlignParams"), + "giant_python.pipeline.extract": ("silo",), + "giant_python.numerics": REMOVED_KERNELS, + "giant_python.numerics.interpolation": ("interp_frame",), + "giant_python.io": REMOVED_IO, + "giant_python.io.hdf5": ("save_struct_h5",), + "giant_python.io.slap2": ("get_online_motion",), + "giant_python.io.tiff": ( + "scanimagetiff_wrapper", + "scanimagetiff_data_wrapper", + ), + "giant_python.gui": ("DrawROIs", "annotate_rois"), +} + + +def _is_stub_function(node): + """Recognize functions containing only a docstring, pass, or ellipsis.""" + body = node.body[1:] if ast.get_docstring(node) is not None else node.body + return all( + isinstance(statement, ast.Pass) + or ( + isinstance(statement, ast.Expr) + and isinstance(statement.value, ast.Constant) + and statement.value.value is Ellipsis + ) + for statement in body + ) + + +def _placeholder_violations(tree, relative_path): + """Return line-numbered AST violations, exempting one exact protocol.""" + parents = { + child: parent + for parent in ast.walk(tree) + for child in ast.iter_child_nodes(parent) + } + violations = [] + for node in ast.walk(tree): + if ( + (isinstance(node, ast.Name) and node.id == "NotImplementedError") + or ( + isinstance(node, ast.Attribute) + and node.attr == "NotImplementedError" + ) + or ( + isinstance(node, ast.alias) + and node.name == "NotImplementedError" + ) + ): + violations.append((node.lineno, "NotImplementedError")) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + if _is_stub_function(node): + parent = parents[node] + is_reader_contract = ( + relative_path == "io/slap2.py" + and isinstance(parent, ast.ClassDef) + and parent.name == "Slap2Reader" + and isinstance(parents[parent], ast.Module) + and any( + isinstance(base, ast.Name) and base.id == "Protocol" + for base in parent.bases + ) + and node.name == "getLineData" + and len(node.body) == 1 + and ast.get_docstring(node) is not None + ) + if not is_reader_contract: + violations.append((node.lineno, "stub: " + node.name)) + return violations + + +class TestRemovedPlaceholders(unittest.TestCase): + """Removal means physical absence and no discoverable replacement API.""" + + def test_deleted_modules_are_physically_absent(self): + """Neither source files nor same-name package shells may remain.""" + for module in REMOVED_MODULES: + path = PACKAGE_ROOT.joinpath(*module.split(".")) + with self.subTest(module=module): + self.assertFalse(path.with_suffix(".py").exists()) + self.assertFalse(path.exists()) + + def test_deleted_modules_cannot_be_discovered(self): + """Import discovery cannot find old modules or compatibility shims.""" + for module in REMOVED_MODULES: + with self.subTest(module=module): + self.assertIsNone(find_spec("giant_python." + module)) + + def test_removed_owner_and_package_exports_are_absent(self): + """Removed symbols cannot survive in owners, exports, or completion.""" + for module_name, names in REMOVED_EXPORTS.items(): + module = import_module(module_name) + for name in names: + with self.subTest(module=module_name, name=name): + self.assertFalse(hasattr(module, name)) + self.assertNotIn(name, getattr(module, "__all__", ())) + self.assertNotIn(name, dir(module)) + + def test_removed_model_and_pipeline_members_are_absent(self): + """Future serialization, stages, and unused session state are gone.""" + for owner in (TrialTable, TrialTable()): + with self.subTest(owner=owner): + self.assertFalse(hasattr(owner, "to_h5")) + for owner in (Pipeline, Pipeline()): + for name in ( + "from_config", + "organize", + "register", + "run_all", + "save_dir", + "align_params", + ): + with self.subTest(owner=owner, name=name): + self.assertFalse(hasattr(owner, name)) + + def test_constructor_exposes_only_supported_options(self): + """Science and routing remain distinct from keyword-only policies.""" + parameters = inspect.signature(Pipeline).parameters + self.assertEqual( + list(parameters), + ["microscope", "params", "execution", "annotations", "scan_mode"], + ) + for name in ("microscope", "params"): + self.assertEqual( + parameters[name].kind, inspect.Parameter.POSITIONAL_OR_KEYWORD + ) + for name in ("execution", "annotations", "scan_mode"): + self.assertEqual( + parameters[name].kind, inspect.Parameter.KEYWORD_ONLY + ) + + def test_removed_constructor_keywords_are_rejected(self): + """Direct and loaded-table constructors reject removed options.""" + table = TrialTable() + for name in ("save_dir", "align_params", "silo_params"): + with self.subTest(keyword=name): + with self.assertRaisesRegex(TypeError, name): + Pipeline(**{name: None}) + with self.assertRaisesRegex(TypeError, name): + Pipeline.from_trial_table(table, **{name: None}) + + def test_production_ast_contains_no_placeholders(self): + """Scan every source function, including private and nested helpers.""" + paths = sorted(PACKAGE_ROOT.rglob("*.py")) + self.assertTrue(paths) + for path in paths: + relative = path.relative_to(PACKAGE_ROOT).as_posix() + tree = ast.parse(path.read_text(encoding="utf-8-sig")) + with self.subTest(path=relative): + self.assertEqual(_placeholder_violations(tree, relative), []) + + def test_real_reader_protocol_is_retained(self): + """The used raw-reader typing declaration is not a removed stub API.""" + from giant_python.io import Slap2Reader + from giant_python.io.slap2 import Slap2Reader as Reader + + self.assertIs(Slap2Reader, Reader) + self.assertTrue(callable(Reader.getLineData)) + self.assertIn("getLineData", Reader.__dict__) + tree = ast.parse( + (PACKAGE_ROOT / "io" / "slap2.py").read_text(encoding="utf-8-sig") + ) + reader = next( + node + for node in tree.body + if isinstance(node, ast.ClassDef) and node.name == "Slap2Reader" + ) + method = next( + node + for node in reader.body + if isinstance(node, ast.FunctionDef) and node.name == "getLineData" + ) + self.assertTrue(_is_stub_function(method)) + self.assertEqual(len(method.body), 1) + self.assertIsNotNone(ast.get_docstring(method)) + self.assertEqual(_placeholder_violations(tree, "io/slap2.py"), []) + + +class TestPlaceholderGuard(unittest.TestCase): + """Exercise the AST guard so forbidden cases cannot silently escape it.""" + + def test_empty_function_variants_are_detected(self): + """Docstrings, pass, ellipsis, async methods, and nested stubs fail.""" + cases = ( + ("def stub(): pass", "stub"), + ("def stub(): ...", "stub"), + ('def stub(): "Only a docstring."', "stub"), + ('def stub():\n "Description."\n pass\n ...', "stub"), + ("async def stub(): ...", "stub"), + ("class Example:\n def method(self): pass", "method"), + ("def outer():\n def inner(): ...", "inner"), + ) + for source, name in cases: + with self.subTest(source=source): + violations = _placeholder_violations( + ast.parse(source), "example.py" + ) + self.assertEqual( + [reason for _, reason in violations], ["stub: " + name] + ) + + def test_not_implemented_error_forms_are_detected(self): + """Bare, constructed, qualified, and aliased error references fail.""" + for source in ( + "raise NotImplementedError", + "raise NotImplementedError('future')", + "raise builtins.NotImplementedError()", + "from builtins import NotImplementedError as FutureError", + ): + with self.subTest(source=source): + self.assertEqual( + _placeholder_violations(ast.parse(source), "example.py"), + [(1, "NotImplementedError")], + ) + + def test_real_functions_and_non_code_mentions_are_allowed(self): + """Ignore comments, strings, and pass within implemented functions.""" + source = dedent('''\ + # NotImplementedError is mentioned, not raised. + def implemented(value): + """NotImplementedError is just documentation here.""" + try: + return value[...] + except IndexError: + pass + return None + + async def asynchronous(): + return "NotImplementedError" + ''') + self.assertEqual( + _placeholder_violations(ast.parse(source), "example.py"), [] + ) + + def test_protocol_exception_is_exact(self): + """No other file, class, method, or empty body gets an exemption.""" + declaration = dedent('''\ + class Slap2Reader(Protocol): + def getLineData(self): + """Vendor reader typing contract.""" + ''') + self.assertEqual( + _placeholder_violations(ast.parse(declaration), "io/slap2.py"), [] + ) + cases = ( + (declaration, "other.py"), + (declaration.replace("Slap2Reader", "OtherReader"), "io/slap2.py"), + (declaration.replace("getLineData", "other"), "io/slap2.py"), + (declaration.replace("(Protocol)", ""), "io/slap2.py"), + ( + declaration.replace( + '"""Vendor reader typing contract."""', "pass" + ), + "io/slap2.py", + ), + (declaration + " ...\n", "io/slap2.py"), + ) + for source, path in cases: + with self.subTest(source=source, path=path): + self.assertEqual( + len(_placeholder_violations(ast.parse(source), path)), 1 + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_numeric_architecture.py b/tests/test_numeric_architecture.py new file mode 100644 index 0000000..a53a54b --- /dev/null +++ b/tests/test_numeric_architecture.py @@ -0,0 +1,296 @@ +"""Canonical kernel ownership, dependency direction, and retired APIs.""" + +import ast +import importlib +import importlib.util +import unittest +from pathlib import Path + +PACKAGE_ROOT = Path(__file__).resolve().parents[1] / "src" / "giant_python" + +# Each kernel has one implementation, in its canonical owning module. +OWNERS = { + "numerics.baseline": ( + "_movmean_nan", + "_compute_f0_column", + "compute_f0", + "assemble_dff", + ), + "numerics.peaks": ( + "gaussian_peaks_integrated", + "_gaussian_peaks_integrated_val_jac", + "_lsq_curvefit", + "_fit_isotropic_then_anisotropic", + "_make_bounds", + "_peak_mask", + "_buffer_mask", + "_process_cc_new_peak", + "_refine_residual_peaks", + "detect_peaks_2d", + "get_act_im_peaks", + ), + "numerics.interpolation": ("nearest_interp",), + "numerics.morphology": ("fast_dilation",), + "progress": ("log", "progress"), + "extraction.band.motion_binning": ( + "bin_motion", + "select_motion_bins", + "bin_motion_yx", + ), + "extraction.band.geometry": ( + "ref_pixs_to_drc", + "build_subsample_matrix_inds", + "_pad_to", + "build_combined_psf", + "threshold_and_crop_psf", + "build_selected_pixel_mask", + "pixel_coords_from_idxs", + "selected_pixels_2d_for_plane", + ), + "extraction.band.operators": ( + "build_sparse_h", + "expand_psf", + "shrink_psf", + "gaussian_kernel_2d", + "build_convolution_matrix", + "build_motion_h_matrices", + ), + "extraction.band.inputs": ( + "load_lookup_table", + "_ref_stack_group", + "find_reference_file", + "load_reference_stack", + "default_psf", + "load_psf", + ), + "extraction.band.background": ( + "_collect_interp_columns", + "_interp_columns", + "build_interp_data", + "baseline_window_frames", + "compute_rolling_baseline", + "assemble_background", + "fit_noise_variance_model", + "compute_residual", + ), + "extraction.band.activity": ( + "_plane_column_maps", + "_valid_sel_cols_for_motion", + "compute_rho", + "mask_high_nan_rho", + "decay_kernel_1d", + "smooth_rho", + ), + "extraction.band.localization": ( + "sel_pix_gaussian_profile", + "sel_pix_patch_profile", + "init_source_params", + "build_a_patches", + "build_x_support_mots", + "project_spatial_profiles", + "_motion_frames", + "solve_phi_motion", + "fit_phi_all_motions", + "multiplicative_nmf", + "fit_gaussian_profiles", + "variance_sortorder", + "reorder_sources", + "compute_source_snr", + "fit_sources", + ), + "extraction.band.summary_images": ( + "compute_mean_image", + "accumulate_activity_image", + "finalize_activity_image", + "phase_matched_nanmedian", + "snake_aligned_nanmedian", + ), + "extraction.band.traces": ( + "compute_high_res_traces", + "get_high_res_traces", + ), + "extraction.band.trial_data": ( + "compute_ds_frames", + "accumulate_superpixel_data", + "read_band_trial_data", + "assemble_lowres_data", + "save_lowres_data", + "load_lowres_data", + ), +} + + +def _source_tree(module): + """Parse a package module without importing its dependencies. + + Parameters + ---------- + module : str + Module path relative to ``giant_python``. + + Returns + ------- + ast.Module + Parsed source tree. + """ + path = PACKAGE_ROOT.joinpath(*module.split(".")).with_suffix(".py") + return ast.parse(path.read_text(encoding="utf-8-sig")) + + +def _imports(module, tree): + """Resolve static import targets, including function-local imports. + + Parameters + ---------- + module : str + Module path relative to ``giant_python``. + tree : ast.Module + Parsed source tree. + + Yields + ------ + str + Absolute imported module or member path. + """ + parent = module.rpartition(".")[0] + package = "giant_python" + ("." + parent if parent else "") + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for alias in node.names: + yield alias.name + elif isinstance(node, ast.ImportFrom): + name = "." * node.level + (node.module or "") + target = importlib.util.resolve_name(name, package) + for alias in node.names: + yield target + "." + alias.name + + +class TestNumericArchitecture(unittest.TestCase): + """Guard canonical ownership and low-level import boundaries.""" + + def test_retired_namespaces_have_no_python_sources(self): + """Permit orphan caches, but never active modules or empty stubs.""" + for namespace in ("bandsilo", "math"): + with self.subTest(namespace=namespace): + self.assertEqual( + list((PACKAGE_ROOT / namespace).rglob("*.py")), [] + ) + + def test_production_imports_do_not_reference_retired_namespaces(self): + """Inspect imports, including relative/local ones, not docstrings.""" + retired = {"giant_python.bandsilo", "giant_python.math"} + for path in PACKAGE_ROOT.rglob("*.py"): + module = ".".join( + path.relative_to(PACKAGE_ROOT).with_suffix("").parts + ) + for target in _imports(module, _source_tree(module)): + with self.subTest(module=module, target=target): + self.assertFalse( + any( + target == name or target.startswith(name + ".") + for name in retired + ) + ) + + def test_public_exports_do_not_restore_retired_namespaces(self): + """Public exports cannot expose the removed compatibility APIs.""" + for name in ( + "giant_python", + "giant_python.numerics", + "giant_python.extraction", + "giant_python.extraction.band", + ): + namespace = importlib.import_module(name) + for retired in ("bandsilo", "math"): + with self.subTest(namespace=name, retired=retired): + self.assertNotIn( + retired, getattr(namespace, "__all__", ()) + ) + + def test_shared_numerics_do_not_import_application_layers(self): + """Shared kernels cannot depend on acquisition, IO, UI, or backends.""" + forbidden = {"h5py", "tifffile", "slap2_utils", "tkinter", "napari"} + for path in (PACKAGE_ROOT / "numerics").glob("*.py"): + module = "numerics." + path.stem + for target in _imports(module, _source_tree(module)): + with self.subTest(module=module, target=target): + self.assertNotIn(target.split(".")[0], forbidden) + self.assertTrue( + not target.startswith("giant_python.") + or target.startswith("giant_python.numerics.") + ) + + def test_band_kernels_do_not_depend_on_legacy_or_workflows(self): + """Keep workflows above kernels and acquisition at input boundaries.""" + blocked = ( + "giant_python.bandsilo.", + "giant_python.math.", + "giant_python.gui.", + "giant_python.pipeline.", + "giant_python.extraction.band.workflow.", + ) + input_boundaries = { + "extraction.band.inputs", + "extraction.band.trial_data", + "extraction.band.traces", + } + for module in OWNERS: + if not module.startswith("extraction.band."): + continue + for target in _imports(module, _source_tree(module)): + with self.subTest(module=module, target=target): + self.assertFalse(target.startswith(blocked)) + if module not in input_boundaries: + self.assertFalse(target.startswith("giant_python.io.")) + self.assertFalse( + target.startswith( + "giant_python.extraction.band.trial_data." + ) + ) + + def test_namespaces_do_not_eagerly_import_backends(self): + """Importing a namespace must not itself import IO or workflows.""" + for module in ( + "numerics.__init__", + "extraction.__init__", + "extraction.band.__init__", + ): + with self.subTest(module=module): + for node in _source_tree(module).body: + self.assertNotIsInstance( + node, (ast.Import, ast.ImportFrom) + ) + + def test_shared_helpers_and_summary_image_dependency(self): + """Acquisition-independent helpers live in numerics, not trial IO.""" + dependencies = { + "extraction.band.summary_images": ( + "giant_python.numerics.morphology.fast_dilation", + ), + "extraction.band.traces": ( + "giant_python.numerics.interpolation.nearest_interp", + ), + "extraction.band.trial_data": ( + "giant_python.io.slap2.open_slap2_file", + ), + } + for module, targets in dependencies.items(): + imports = set(_imports(module, _source_tree(module))) + for target in targets: + with self.subTest(module=module, target=target): + self.assertIn(target, imports) + + def test_kernels_have_one_actual_definition(self): + """Scan all production modules, not just the expected owner list.""" + definitions = {} + for path in PACKAGE_ROOT.rglob("*.py"): + module = ".".join( + path.relative_to(PACKAGE_ROOT).with_suffix("").parts + ) + for node in _source_tree(module).body: + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + definitions.setdefault(node.name, []).append(module) + for owner, names in OWNERS.items(): + for name in names: + with self.subTest(owner=owner, name=name): + self.assertEqual(definitions.get(name), [owner]) diff --git a/tests/test_numeric_edges.py b/tests/test_numeric_edges.py new file mode 100644 index 0000000..069ec10 --- /dev/null +++ b/tests/test_numeric_edges.py @@ -0,0 +1,184 @@ +"""Value-based regressions for previously uncovered numerical edge paths.""" + +import unittest +from unittest.mock import patch + +import numpy as np + +from giant_python.extraction.band import background, operators, summary_images +from giant_python.numerics import peaks + + +class TestOperatorEdges(unittest.TestCase): + """Protect normalization, centering, and nonnegative kernel weights.""" + + def test_identity_shrink_preserves_normalized_psf(self): + """Unit scales preserve a positive PSF up to sum normalization.""" + psf = np.arange(1, 26, dtype=np.float32).reshape(5, 5) + original = psf.copy() + actual = operators.shrink_psf(psf, scale_y=1, scale_x=1) + np.testing.assert_allclose(actual, psf / psf.sum(), rtol=1e-6) + np.testing.assert_array_equal(psf, original) + self.assertEqual(actual.dtype, np.float32) + + def test_shrink_narrows_only_requested_axis(self): + """An x-only shrink narrows the x moment without moving the center.""" + coords = np.arange(-6, 7, dtype=float) + psf = np.exp(-(coords[:, None] ** 2 + coords[None, :] ** 2) / 8) + actual = operators.shrink_psf(psf, scale_y=1, scale_x=0.5) + original = psf / psf.sum() + self.assertEqual(actual.shape, psf.shape) + self.assertTrue(np.all(actual >= 0)) + self.assertAlmostEqual(float(actual.sum()), 1, places=6) + np.testing.assert_allclose(actual, actual[::-1, ::-1], atol=1e-7) + self.assertLess( + np.sum(actual * coords[None, :] ** 2), + np.sum(original * coords[None, :] ** 2), + ) + np.testing.assert_allclose( + actual.sum(axis=1), original.sum(axis=1), atol=1e-6 + ) + + def test_nonpositive_psf_remains_zero_without_division(self): + """Clip negative weights and avoid normalizing zero total mass.""" + for value in (0, -1): + with self.subTest(value=value): + actual = operators.shrink_psf(np.full((5, 5), value)) + np.testing.assert_array_equal(actual, np.zeros((5, 5))) + self.assertEqual(actual.dtype, np.float32) + + def test_gaussian_kernel_matches_separable_weights(self): + """Use ceil-truncated support and a centered unit-mass Gaussian.""" + sigma, truncate = 0.8, 2.0 + kernel, center = operators.gaussian_kernel_2d(sigma, truncate) + self.assertEqual(center, (2, 2)) + actual = kernel.numpy() + coords = np.arange(-2, 3) + weights = np.exp(-(coords**2) / (2 * sigma**2)) + weights /= weights.sum() + np.testing.assert_allclose( + actual, np.outer(weights, weights), rtol=1e-6 + ) + self.assertEqual(actual.dtype, np.float32) + self.assertEqual( + np.unravel_index(actual.argmax(), actual.shape), center + ) + self.assertAlmostEqual(float(actual.sum()), 1, places=6) + + +class TestBackgroundEdges(unittest.TestCase): + """Check fallback medians, minimal windows, and interpolation rejection.""" + + def test_pandas_fallback_ignores_nan_and_preserves_dtype(self): + """Fallback clips odd windows and preserves empty-window NaNs.""" + data = np.array( + [[1, np.nan, 3, 100, 5], [np.nan] * 5], dtype=np.float32 + ) + original = data.copy() + with patch.object(background, "bn", None): + actual = background.compute_rolling_baseline(data, 3) + expected = np.array([[1, 2, 51.5, 5, 52.5], [np.nan] * 5]) + np.testing.assert_allclose(actual, expected, equal_nan=True) + np.testing.assert_array_equal(data, original) + self.assertEqual(actual.dtype, data.dtype) + + def test_single_frame_window_preserves_samples(self): + """Nonpositive and unit windows clamp to one without padding.""" + data = np.array([[1, np.nan, 7]], dtype=np.float32) + for window in (-2, 0, 1): + with self.subTest(window=window): + actual = background.compute_rolling_baseline(data, window) + np.testing.assert_array_equal(actual, data) + self.assertEqual(actual.dtype, data.dtype) + + def test_internal_interpolation_rejects_unknown_method(self): + """Reject invalid interpolation modes before mutating the output.""" + out = np.full((1, 1), np.nan, dtype=np.float32) + with self.assertRaisesRegex(ValueError, "linear.*cubic"): + background._interp_columns( + out, + {}, + {}, + {}, + np.array([0]), + np.ones((1, 1)), + np.array([0]), + np.float32, + method="nearest", + ) + self.assertTrue(np.isnan(out).all()) + + +class TestMedianGeometryEdges(unittest.TestCase): + """Reject malformed geometry instead of producing plausible medians.""" + + def test_phase_filter_rejects_invalid_geometry(self): + """Validate image, coordinate, depth, and selected-index contracts.""" + defaults = dict( + image=np.ones((1, 3, 3)), + valid_pix_idxs=np.arange(9), + ref_r=np.array([1]), + ref_c=np.array([1]), + ) + cases = [ + ({"image": np.ones((3, 3))}, "non-empty shape"), + ({"image": np.ones((1, 0, 3))}, "non-empty shape"), + ({"ref_r": []}, "non-empty and equally sized"), + ({"ref_c": [0, 1]}, "non-empty and equally sized"), + ({"ref_r": [np.nan]}, "reference rows must be finite"), + ({"ref_d": [0, 0]}, "same size"), + ({"ref_c": [0.5]}, "finite integers"), + ({"ref_d": [np.inf]}, "finite integers"), + ({"ref_d": [0.5]}, "finite integers"), + ({"valid_pix_idxs": [[0]]}, "one-dimensional integer array"), + ({"valid_pix_idxs": [0.5]}, "one-dimensional integer array"), + ({"valid_pix_idxs": [-1]}, "out-of-bounds"), + ({"valid_pix_idxs": [9]}, "out-of-bounds"), + ] + for overrides, message in cases: + with self.subTest(overrides=overrides): + with self.assertRaisesRegex(ValueError, message): + summary_images.phase_matched_nanmedian( + **(defaults | overrides) + ) + + def test_snake_filter_rejects_invalid_geometry(self): + """Require odd windows and at least one matched in-bounds center.""" + defaults = dict( + image=np.ones((1, 3, 3)), + valid_pix_idxs=np.arange(9), + ref_r=np.array([1]), + ref_c=np.array([1]), + ) + cases = [ + ({"height": 0}, "positive odd integers"), + ({"width": 2}, "positive odd integers"), + ({"ref_r": []}, "non-empty and equally sized"), + ({"ref_c": [0, 1]}, "non-empty and equally sized"), + ({"ref_c": [-1]}, "no reference centers"), + ({"ref_c": [3]}, "no reference centers"), + ] + for overrides, message in cases: + with self.subTest(overrides=overrides): + with self.assertRaisesRegex(ValueError, message): + summary_images.snake_aligned_nanmedian( + **(defaults | overrides) + ) + + +class TestPeakBoundsEdges(unittest.TestCase): + """Protect the shared-width initialization's feasible parameter domain.""" + + def test_disjoint_width_bounds_rejected_before_fitting(self): + """Disjoint y/x intervals cannot initialize an isotropic peak.""" + theta = np.array([[1, 0, 0, 0.5, 2.5]], dtype=float) + original = theta.copy() + lower = np.array([0, -1, -1, 0.25, 2]) + upper = np.array([2, 1, 1, 1, 3]) + with patch.object(peaks, "_lsq_curvefit") as fit: + with self.assertRaisesRegex(ValueError, "overlapping y/x"): + peaks._fit_isotropic_then_anisotropic( + theta, np.zeros((2, 1)), np.zeros(1), lower, upper + ) + fit.assert_not_called() + np.testing.assert_array_equal(theta, original) diff --git a/tests/test_package_metadata.py b/tests/test_package_metadata.py index 4b89eb4..10da919 100644 --- a/tests/test_package_metadata.py +++ b/tests/test_package_metadata.py @@ -2,19 +2,110 @@ import ast import unittest +from importlib import import_module +from importlib.resources import files +from importlib.util import find_spec from pathlib import Path -PACKAGE_INIT = ( - Path(__file__).resolve().parents[1] - / "src" - / "giant_python" - / "__init__.py" -) +from setuptools import find_packages + +SOURCE_ROOT = Path(__file__).resolve().parents[1] / "src" +PACKAGE_INIT = SOURCE_ROOT / "giant_python" / "__init__.py" class PackageMetadataTest(unittest.TestCase): """Tests for package metadata declarations.""" + def test_canonical_packages_are_discoverable(self): + """Source discovery includes canonical owners, not retired packages.""" + packages = set(find_packages(where=str(SOURCE_ROOT))) + self.assertTrue( + { + "giant_python", + "giant_python.extraction", + "giant_python.extraction.band", + "giant_python.numerics", + }.issubset(packages) + ) + for package in packages: + with self.subTest(package=package): + self.assertFalse( + package == "giant_python.bandsilo" + or package.startswith("giant_python.bandsilo.") + or package == "giant_python.math" + or package.startswith("giant_python.math.") + ) + + def test_summary_modules_have_explicit_names(self): + """Summary owners keep public exports without old module shims.""" + import giant_python + + for owner, symbols in ( + ("models", ("ExperimentSummary", "PathSummary", "FrameInfo")), + ("io", ("read_summary", "write_summary")), + ): + package = import_module(f"giant_python.{owner}") + module = import_module(f"giant_python.{owner}.experiment_summary") + for name in symbols: + with self.subTest(owner=owner, symbol=name): + symbol = getattr(module, name) + self.assertEqual(symbol.__module__, module.__name__) + self.assertIs(getattr(package, name), symbol) + if owner == "models": + self.assertIs(getattr(giant_python, name), symbol) + self.assertIsNone(find_spec(f"giant_python.{owner}.experiment")) + old_path = SOURCE_ROOT / "giant_python" / owner / "experiment.py" + self.assertFalse(old_path.exists()) + + def test_band_and_gui_modules_have_explicit_names(self): + """Renamed modules own their functions without forwarding shims.""" + renames = { + "giant_python.extraction.band.results": ( + "result_assembly", + "assemble_path_summary", + ), + "giant_python.extraction.band.motion": ( + "motion_binning", + "bin_motion", + ), + "giant_python.gui.parameters": ( + "parameters_gui", + "run_parameter_gui", + ), + } + for old_name, (new_name, symbol) in renames.items(): + with self.subTest(module=new_name): + parent = old_name.rsplit(".", 1)[0] + module = import_module(f"{parent}.{new_name}") + self.assertEqual( + getattr(module, symbol).__module__, module.__name__ + ) + self.assertIsNone(find_spec(old_name)) + old_path = SOURCE_ROOT.joinpath( + *old_name.split(".") + ).with_suffix(".py") + self.assertFalse(old_path.exists()) + + def test_psf_assets_are_available_as_package_resources(self): + """Every bundled dilation template is readable without a build.""" + assets = files("giant_python").joinpath("assets", "psfs") + dilations = (*range(5, 32, 2), 41, 51) + for dilation in dilations: + name = f"dil-{dilation:02d}.tif" + with self.subTest(asset=name): + resource = assets.joinpath(name) + self.assertTrue(resource.is_file()) + with resource.open("rb") as stream: + self.assertIn( + stream.read(4), + ( + b"II\x2a\x00", + b"MM\x00\x2a", + b"II\x2b\x00", + b"MM\x00\x2b", + ), + ) + def test_init_module_is_valid_and_has_single_version_line(self): """Only one ``__version__`` line may exist so CI sed bumps stay safe. diff --git a/tests/test_parallel.py b/tests/test_parallel.py index 4312163..e9be201 100644 --- a/tests/test_parallel.py +++ b/tests/test_parallel.py @@ -35,6 +35,13 @@ def test_desc_shows_progress_bar(self): map_trials(_double, [1, 2, 3], desc="doubling"), [2, 4, 6] ) + def test_parallel_progress_preserves_order(self): + """Progress wraps completed worker results without changing order.""" + self.assertEqual( + map_trials(_double, [3, 1, 2], n_workers=2, desc="parallel"), + [6, 2, 4], + ) + if __name__ == "__main__": unittest.main() diff --git a/tests/test_parameter_gui.py b/tests/test_parameter_gui.py new file mode 100644 index 0000000..dc93654 --- /dev/null +++ b/tests/test_parameter_gui.py @@ -0,0 +1,105 @@ +"""Headless checks for the parameter form's canonical split-option API.""" + +import sys +import unittest +from unittest.mock import MagicMock, patch + +from giant_python.gui.parameters_gui import run_parameter_gui +from giant_python.models.params import ( + AnnotationOptions, + BandSiloParams, + ExecutionOptions, +) + + +class TestParameterGui(unittest.TestCase): + """Exercise form callbacks without creating a desktop window.""" + + def setUp(self): + """Replace toolkit widgets with controlled variables and callbacks.""" + self.tk = MagicMock() + self.root = self.tk.Tk.return_value + self.variables = [] + self.buttons = {} + self.tk.StringVar.side_effect = self._string_var + self.tk.BooleanVar.side_effect = self._boolean_var + self.tk.ttk.Button.side_effect = self._button + self.modules = patch.dict(sys.modules, {"tkinter": self.tk}) + self.modules.start() + self.addCleanup(self.modules.stop) + + def _string_var(self, *, value): + """Retain each field variable so simulated users can edit it.""" + variable = MagicMock() + variable.get.return_value = value + self.variables.append(variable) + return variable + + def _boolean_var(self, *, value): + """Track the ROI inclusion control separately from text fields.""" + self.roi_var = MagicMock() + self.roi_var.get.return_value = value + return self.roi_var + + def _button(self, parent, *, text, command): + """Capture form actions instead of binding real toolkit events.""" + self.buttons[text] = command + return MagicMock() + + def test_success_returns_fresh_split_options(self): + """Scientific, execution, and annotation controls stay independent.""" + params = BandSiloParams(peakth=8) + run = ExecutionOptions(max_workers=2, verbose=True, max_trials=4) + roi = AnnotationOptions(interactive=False, operator="original") + + def interact(): + """Simulate edits to a scientific field and both policy groups.""" + self.variables[0].get.return_value = "125" + self.variables[8].get.return_value = "3" + self.variables[9].get.return_value = "reviewer" + self.roi_var.get.return_value = True + self.buttons["OK"]() + + self.root.mainloop.side_effect = interact + result = run_parameter_gui(params, execution=run, annotations=roi) + assert result is not None + science, execution, annotations = result + self.assertEqual(science.analyze_hz, 125) + self.assertEqual(science.peakth, 8) + self.assertEqual(execution, ExecutionOptions(3, True, 4)) + self.assertEqual( + annotations, AnnotationOptions(True, False, "reviewer") + ) + self.assertEqual(params.analyze_hz, 100) + self.assertEqual(run.max_workers, 2) + self.assertFalse(roi.enabled) + self.assertEqual(roi.operator, "original") + for supplied, returned in zip((params, run, roi), result): + self.assertIsNot(supplied, returned) + self.root.destroy.assert_called_once() + self.tk.messagebox.showerror.assert_not_called() + + def test_cancel_returns_none(self): + """Cancel does not manufacture a combined parameter result.""" + self.root.mainloop.side_effect = lambda: self.buttons["Cancel"]() + self.assertIsNone(run_parameter_gui()) + self.root.destroy.assert_called_once() + + def test_invalid_input_shows_error_without_accepting(self): + """Invalid scientific values leave the form open for correction.""" + + def interact(): + """Try a nonpositive rate, then cancel the still-open form.""" + self.variables[0].get.return_value = "0" + self.buttons["OK"]() + self.root.destroy.assert_not_called() + self.tk.messagebox.showerror.assert_called_once() + self.buttons["Cancel"]() + + self.root.mainloop.side_effect = interact + self.assertIsNone(run_parameter_gui()) + self.root.destroy.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_path_summary.py b/tests/test_path_summary.py new file mode 100644 index 0000000..6f01061 --- /dev/null +++ b/tests/test_path_summary.py @@ -0,0 +1,374 @@ +"""Nonzero two-path assembly, model, and emitted-schema round-trip coverage.""" + +import tempfile +import unittest +from pathlib import Path + +import h5py +import numpy as np + +from giant_python.extraction.band.inputs import load_alignment_data_h5 +from giant_python.extraction.band.result_assembly import ( + PathResult, + assemble_path_summary, +) +from giant_python.extraction.band.types import TrialTraceResult +from giant_python.io.experiment_summary import ( + read_summary, + write_summary, +) +from giant_python.models.experiment_summary import ( + FRAME_INFO_KEYS, + ExperimentSummary, + FrameInfo, +) +from tests.test_bandsilo_pipeline import _path_result + + +class _ArrayTensor: + """Minimal tensor interface for assembly without a torch import.""" + + def __init__(self, array): + """Keep the supplied array unchanged.""" + self.array = array + + def numpy(self): + """Expose the backing array, like the CPU tensor producer.""" + return self.array + + def __getitem__(self, key): + """Return a tensor-like slice for source-parameter indexing.""" + return _ArrayTensor(self.array[key]) + + +def _write_alignment(path, index): + """Write signed fractional displacement knots with a missing sample.""" + saved = { + "motionDSr": np.array([1.25, -2.5, np.nan]) + index, + "motionDSc": np.array([-3.5, 0.75, 2.25]) - index, + "motionDSz": np.array([0.5, -1.25, 3.0]) + 2 * index, + } + with h5py.File(path, "w") as f: + f["row_major"] = 1 + f["DSframes"] = np.array([[1, 5, 9]]) + f["numChannels"] = 2 + f["alignHz"] = 10.0 + for key, value in saved.items(): + f[key] = value.reshape(-1, 1) + online = f.create_group("slap2") + for axis in ("X", "Y", "Z"): + online[f"onlineMotion{axis}shift"] = np.array([[2, -3, 4]]) + return saved + + +def _result(index, alignment, roi_state="present"): + """Build two stitched trials with distinct path shapes and ROI policies.""" + n_sources, channels, nz, rows, cols = ( + (2, 2, 1, 3, 4) if index == 0 else (3, 1, 2, 2, 3) + ) + frames = np.arange(1, 10, dtype=float) + # Match the existing band linear/clamped interpolation, not MATLAB PCHIP. + offline = tuple( + np.interp(frames, alignment["DSframes"], alignment[key]) + for key in ("motionDSr", "motionDSc", "motionDSz") + ) + online = tuple(np.arange(9, dtype=np.int16) + axis for axis in (3, -2, 1)) + n_rois = 1 if roi_state == "present" else 0 + traces = [] + for section in (slice(0, 4), slice(4, 9)): + n_frames = frames[section].size + signal = ( + np.arange(n_frames * n_sources, dtype=np.float32).reshape( + n_frames, n_sources + ) + / 10 + ) + traces.append( + TrialTraceResult( + signal, + np.full_like(signal, 5.0), + frames[section], + np.arange(3), + np.full((n_frames, channels), index + 2.0, dtype=np.float32), + tuple(axis[section] for axis in offline), + tuple(axis[section] for axis in online), + np.full( + (n_frames, n_rois, channels), index + 7.0, dtype=np.float32 + ), + ) + ) + masks = [np.ones((nz, rows, cols), dtype=bool)] if n_rois else [] + return PathResult( + a=_ArrayTensor( + np.arange(nz * rows * cols * n_sources, dtype=np.float32).reshape( + -1, n_sources + ) + ), + source_params=_ArrayTensor( + np.arange(n_sources * 6, dtype=np.float32).reshape(n_sources, 6) + ), + source_snr=np.arange(1, n_sources + 1, dtype=np.float32), + act_im=np.full((nz, rows, cols), index + 0.25, dtype=np.float32), + mean_im=np.full( + (channels, nz, rows, cols), index + 4.5, dtype=np.float32 + ), + act_im_peaks=np.zeros((n_sources, 3), dtype=np.int32), + trace_results=traces, + z_depths=np.arange(nz).reshape(-1, 1), + num_fast_zs=nz, + dmd_pixels_per_column=rows, + dmd_pixels_per_row=cols, + num_channels=channels, + denoise_window=3, + baseline_window=5, + draw_user_rois=roi_state != "absent", + soma_masks=masks if roi_state != "absent" else None, + soma_labels=["ROI μ"] if n_rois else [], + yx_shape=(rows, cols), + ) + + +def _datasets(path): + """Return raw dataset values and dtypes for complete schema comparison.""" + values = {} + with h5py.File(path, "r") as f: + + def collect(name, item): + """Collect datasets, retaining stored shape/dtype and values.""" + if isinstance(item, h5py.Dataset): + values[name] = (item[()], item.dtype) + + f.visititems(collect) + return values + + +class TestPathSummaryRoundTrip(unittest.TestCase): + """Per-path data and corrected public shifts survive HDF5 round trips.""" + + def test_nonzero_two_path_model_and_schema_roundtrip(self): + """Raw -> normalized -> interpolated -> assembled -> model -> disk.""" + with tempfile.TemporaryDirectory() as dr: + results, paths, expected = [], [], [] + for index in range(2): + alignment_path = Path(dr) / f"alignment{index}.h5" + saved = _write_alignment(alignment_path, index) + result = _result( + index, + load_alignment_data_h5(alignment_path), + "present" if index == 0 else "empty", + ) + results.append(result) + paths.append(assemble_path_summary(result, index)) + expected.append( + { + axis: np.interp( + np.arange(1, 10), [1, 5, 9], saved[key] + ) + for axis, key in ( + ("Y", "motionDSr"), + ("X", "motionDSc"), + ("Z", "motionDSz"), + ) + } + ) + summary = ExperimentSummary( + params={ + "operator": "μ", + "numChannels": 2, + "draw_user_rois": True, + }, + paths=paths, + ) + self.assertEqual([p.path_index for p in summary.paths], [0, 1]) + self.assertEqual([len(p.sources) for p in summary.paths], [2, 3]) + self.assertEqual([len(p.user_rois) for p in summary.paths], [1, 0]) + self.assertTrue(all(p.annotation_enabled for p in summary.paths)) + for index, path in enumerate(summary.paths): + self.assertIsInstance(path.frame_info, FrameInfo) + self.assertIs( + path.frame_info.to_dict()["offlineXshifts"], + path.frame_info.offlineXshifts, + ) + self.assertTrue( + np.shares_memory( + path.sources[0].profile, results[index].a.numpy() + ) + ) + for axis in ("X", "Y", "Z"): + np.testing.assert_allclose( + getattr( + path.frame_info, f"offline{axis}shifts" + ).ravel(), + expected[index][axis], + equal_nan=True, + ) + for axis, position in (("X", 1), ("Y", 0), ("Z", 2)): + np.testing.assert_array_equal( + getattr( + path.frame_info, f"online{axis}shifts" + ).ravel(), + np.concatenate( + [ + r.online_shifts[position] + for r in results[index].trace_results + ] + ), + ) + # Assembling output did not invert the private input arrays. + np.testing.assert_allclose( + -np.concatenate( + [ + r.reference_offsets[1] + for r in results[index].trace_results + ] + ), + expected[index]["X"], + equal_nan=True, + ) + model_file = Path(dr) / "model.h5" + roundtrip_file = Path(dr) / "roundtrip.h5" + self.assertIsNone(write_summary(summary, model_file)) + reloaded = ExperimentSummary.from_h5(model_file) + self.assertIsNone(reloaded.to_h5(roundtrip_file)) + original = _datasets(model_file) + for filename in (roundtrip_file,): + actual = _datasets(filename) + self.assertEqual(set(actual), set(original)) + for key, (value, dtype) in original.items(): + with self.subTest(file=filename.name, dataset=key): + self.assertEqual(actual[key][1], dtype) + np.testing.assert_array_equal(actual[key][0], value) + for index, path in enumerate(reloaded.paths): + self.assertEqual(path.path_index, index) + np.testing.assert_array_equal( + path.visualizations.mean_im, results[index].mean_im + ) + np.testing.assert_array_equal( + path.global_f, paths[index].global_f + ) + np.testing.assert_array_equal( + path.z_depths, results[index].z_depths + ) + with h5py.File(model_file, "r") as f: + self.assertEqual(f["Path2/user_rois/mask"].shape, (0, 2, 2, 3)) + self.assertEqual(f["Path2/user_rois/F"].shape, (0, 1, 9)) + self.assertEqual(f["Path2/user_rois/labels"].shape, (0, 1)) + for key in FRAME_INFO_KEYS: + self.assertEqual(f[f"Path1/frame_info/{key}"].ndim, 2) + + def test_explicit_paths_and_typed_frame_info_only(self): + """No aggregate, single-path convenience or mapping APIs remain.""" + data = { + "DSframes": [1, 5, 9], + **{ + k: np.zeros(3) for k in ("motionDSr", "motionDSc", "motionDSz") + }, + } + first = assemble_path_summary(_result(0, data), path_index=0) + second = assemble_path_summary(_result(1, data), path_index=1) + summary = ExperimentSummary(paths=[first, second]) + for model in ( + summary, + ExperimentSummary(paths=[first]), + ExperimentSummary(), + ): + for name in ( + "frame_info", + "visualizations", + "z_depths", + "global_f", + "sources", + "user_rois", + "_single_path", + ): + with self.subTest(paths=len(model.paths), name=name): + self.assertFalse(hasattr(model, name)) + self.assertIs(summary.paths[1].sources, second.sources) + self.assertIs(summary.paths[0].user_rois, first.user_rois) + for name in ("__getitem__", "__iter__", "__len__"): + self.assertFalse(hasattr(first.frame_info, name)) + self.assertEqual(set(first.frame_info.to_dict()), set(FRAME_INFO_KEYS)) + + def test_absent_rois_and_overwrite_semantics(self): + """Absent differs from empty; rewrites retain old params.""" + data = { + "DSframes": [1, 5, 9], + **{ + k: np.zeros(3) for k in ("motionDSr", "motionDSc", "motionDSz") + }, + } + first = assemble_path_summary(_result(0, data, "empty"), 0) + second = assemble_path_summary(_result(1, data, "present"), 1) + with tempfile.TemporaryDirectory() as dr: + path = Path(dr) / "summary.h5" + write_summary( + ExperimentSummary( + paths=[first, second], params={"operator": "original"} + ), + path, + ) + replacement = assemble_path_summary(_result(0, data, "absent"), 0) + write_summary( + ExperimentSummary( + paths=[replacement], params={"operator": "new"} + ), + path, + ) + loaded = read_summary(path) + self.assertFalse(loaded.paths[0].annotation_enabled) + self.assertTrue(loaded.paths[1].annotation_enabled) + self.assertEqual(loaded.params["operator"], "original") + with h5py.File(path, "r") as f: + self.assertNotIn("user_rois", f["Path1"]) + self.assertIn("user_rois", f["Path2"]) + + def test_empty_sources_retain_shape_and_dtype(self): + """Numerical assembly and codecs retain empty tensor layout.""" + empty = assemble_path_summary(_path_result(n_sources=0), 2) + summary = ExperimentSummary(paths=[empty]) + with tempfile.TemporaryDirectory() as dr: + path = Path(dr) / "empty.h5" + summary.to_h5(path) + restored = read_summary(path) + self.assertEqual(restored.paths[0].path_index, 2) + self.assertEqual(restored.paths[0].sources, []) + for key, arr in restored.paths[0]._empty_source_arrays.items(): + expected = empty._empty_source_arrays[key] + self.assertEqual(arr.shape, expected.shape) + self.assertEqual(arr.dtype, expected.dtype) + + def test_column_major_summary_decodes_without_sign_conversion(self): + """The summary codec uses the same 3D/4D axis convention as raw IO.""" + data = { + "DSframes": [1, 5, 9], + "motionDSr": np.array([1.25, -3.5, 2.0]), + "motionDSc": np.array([-2.5, 0.25, 4.0]), + "motionDSz": np.array([0.5, 1.5, -0.75]), + } + summary = ExperimentSummary( + paths=[ + assemble_path_summary(_result(0, data, "present"), 0), + assemble_path_summary(_result(1, data, "empty"), 1), + ] + ) + with tempfile.TemporaryDirectory() as dr: + original = Path(dr) / "row.h5" + matlab = Path(dr) / "column.h5" + restored = Path(dr) / "restored.h5" + write_summary(summary, original) + datasets = _datasets(original) + with h5py.File(matlab, "w") as f: + for name, (value, dtype) in datasets.items(): + array = np.asarray(value) + if name == "row_major": + array = np.asarray(0) + elif array.ndim >= 2: + array = array.T + f.create_dataset(name, data=array, dtype=dtype) + loaded = read_summary(matlab) + write_summary(loaded, restored) + rewritten = _datasets(restored) + for name, (value, dtype) in datasets.items(): + with self.subTest(dataset=name): + self.assertEqual(rewritten[name][1], dtype) + np.testing.assert_array_equal(rewritten[name][0], value) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py index 0576c9d..0b9cf0d 100644 --- a/tests/test_pipeline.py +++ b/tests/test_pipeline.py @@ -1,149 +1,11 @@ -"""Tests for the pipeline subpackage (stage functions, classes, facade).""" +"""Tests for the implemented annotation and extraction pipeline facade.""" import unittest +from pathlib import Path from unittest import mock -from giant_python.models import AlignParams, SiloParams -from giant_python.pipeline import ( - BandRegistration, - BandSourceExtractor, - MotionCorrector, - MultiRoiRegistration, - Pipeline, - SourceExtractor, - StandardSourceExtractor, - StripRegistration, - TrialTableBuilder, - band_registration, - build_trial_table, - build_trial_table_slap2, - multi_roi_registration, - silo, - strip_registration, - verify_files, -) - - -class TestStageFunctions(unittest.TestCase): - """The porting-faithful stage functions raise NotImplementedError.""" - - def test_build_trial_table(self): - """build_trial_table raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - build_trial_table() - - def test_build_trial_table_slap2(self): - """build_trial_table_slap2 raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - build_trial_table_slap2() - - def test_verify_files(self): - """verify_files raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - verify_files("trial_table.h5", "/some/dir", {}) - - def test_multi_roi_registration(self): - """multi_roi_registration raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - multi_roi_registration() - - def test_strip_registration(self): - """strip_registration raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - strip_registration() - - def test_band_registration(self): - """band_registration raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - band_registration() - - def test_silo(self): - """silo raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - silo() - - -class TestStageClasses(unittest.TestCase): - """Stage classes construct; their run methods raise NotImplementedError.""" - - def test_trial_table_builder(self): - """TrialTableBuilder.run raises NotImplementedError.""" - builder = TrialTableBuilder("slap2") - with self.assertRaises(NotImplementedError): - builder.run("/data", "/results") - - def test_motion_corrector_for_microscope(self): - """MotionCorrector.for_microscope raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - MotionCorrector.for_microscope("slap2", AlignParams()) - - def test_motion_corrector_run(self): - """MotionCorrector.run raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - MotionCorrector().run(None) - - def test_registration_backends_run(self): - """Each registration backend's run raises NotImplementedError.""" - for backend in ( - MultiRoiRegistration, - StripRegistration, - BandRegistration, - ): - with self.assertRaises(NotImplementedError): - backend().run(None) - - def test_source_extractor(self): - """SourceExtractor.run raises NotImplementedError.""" - extractor = SourceExtractor(SiloParams()) - with self.assertRaises(NotImplementedError): - extractor.run(None) - - def test_source_extractor_for_scan_mode(self): - """for_scan_mode dispatches by scan mode and rejects unknown modes.""" - self.assertIsInstance( - SourceExtractor.for_scan_mode("band", SiloParams()), - BandSourceExtractor, - ) - self.assertIsInstance( - SourceExtractor.for_scan_mode("standard", SiloParams()), - StandardSourceExtractor, - ) - with self.assertRaises(ValueError): - SourceExtractor.for_scan_mode("bogus", SiloParams()) - - def test_standard_backend_run_not_implemented(self): - """The standard backend's run is still a stub.""" - with self.assertRaises(NotImplementedError): - StandardSourceExtractor().run(None) - - def test_band_backend_run_delegates(self): - """The band backend resolves the trial-table path and delegates. - - The heavy driver is patched so only the wiring is exercised. - """ - import giant_python.bandsilo.pipeline as bpl - from giant_python.models import TrialTable - - captured = {} - - def _fake(path, params_in=None): - """Capture the delegated call and return a sentinel.""" - captured["path"] = path - captured["params"] = params_in - return "SUMMARY" - - orig = bpl.extract_band_sources - bpl.extract_band_sources = _fake - try: - params = SiloParams(scan_mode="band") - extractor = BandSourceExtractor(params) - out = extractor.run(TrialTable(savedr="/results")) - finally: - bpl.extract_band_sources = orig - - self.assertEqual(out, "SUMMARY") - self.assertEqual(captured["path"].name, "trial_table.h5") - self.assertIs(captured["params"], params) +from giant_python.models import BandSiloParams, ExperimentSummary, TrialTable +from giant_python.pipeline import Pipeline class TestPipelineFacade(unittest.TestCase): @@ -151,109 +13,69 @@ class TestPipelineFacade(unittest.TestCase): def setUp(self): """Construct a pipeline for each test.""" - self.pipe = Pipeline(microscope="slap2", save_dir="/results") + self.pipe = Pipeline(microscope="slap2") def test_holds_session_state(self): """The facade holds config and starts with no data.""" self.assertEqual(self.pipe.microscope, "slap2") self.assertIsNone(self.pipe.tt) self.assertIsNone(self.pipe.summary) - self.assertEqual(self.pipe.silo_params.microscope, "slap2") + self.assertIsInstance(self.pipe.params, BandSiloParams) + self.assertEqual(self.pipe.scan_mode, "band") + self.assertFalse(hasattr(self.pipe, "silo_params")) - def test_organize_delegates(self): - """organize delegates to the builder and stores its trial table.""" - with mock.patch( - "giant_python.pipeline.pipeline.TrialTableBuilder" - ) as builder: - builder.return_value.run.return_value = "TT" - result = self.pipe.organize("/data") - self.assertIs(result, self.pipe) - self.assertEqual(self.pipe.tt, "TT") - - def test_register_delegates(self): - """register delegates to the motion corrector and updates the table.""" - with mock.patch( - "giant_python.pipeline.pipeline.MotionCorrector" - ) as corrector: - corrector.for_microscope.return_value.run.return_value = "TT2" - self.pipe.tt = "TT" - result = self.pipe.register() - self.assertIs(result, self.pipe) - self.assertEqual(self.pipe.tt, "TT2") + def test_params_keyword(self): + """Only the canonical scientific keyword is accepted.""" + params = BandSiloParams() + self.assertIs(Pipeline(params=params).params, params) + with self.assertRaises(TypeError): + Pipeline(silo_params=params) def test_annotate_delegates(self): - """A non-band scan mode uses the generic annotation GUI.""" - self.pipe.silo_params.scan_mode = "standard" - with mock.patch("giant_python.gui.annotate_rois") as annotate_rois: - result = self.pipe.annotate() - annotate_rois.assert_called_once() - self.assertIs(result, self.pipe) + """Explicit standard annotation fails instead of pretending to work.""" + self.pipe.scan_mode = "standard" + self.pipe.tt = TrialTable() + with self.assertRaisesRegex(ValueError, "standard"): + self.pipe.annotate() def test_annotate_band_delegates(self): """A band scan mode uses the BandSILo standalone annotation step.""" - self.pipe.silo_params.scan_mode = "band" - self.pipe.tt = mock.Mock(savedr="/results") + self.pipe.tt = TrialTable(savedr=Path("/results")) with mock.patch( - "giant_python.bandsilo.annotate.annotate_band_rois" + "giant_python.pipeline.pipeline.annotate_rois", + return_value="annotations.h5", ) as annotate_band: result = self.pipe.annotate() annotate_band.assert_called_once() - called_path = annotate_band.call_args[0][0] - self.assertEqual(called_path.name, "trial_table.h5") + self.assertIs(annotate_band.call_args[0][0], self.pipe.tt) + self.assertTrue(self.pipe.annotations.enabled) self.assertIs(result, self.pipe) def test_extract_delegates(self): """extract dispatches by scan mode and stores the summary.""" - self.pipe.silo_params.scan_mode = "band" + self.pipe.tt = TrialTable() + summary = ExperimentSummary() with mock.patch( - "giant_python.pipeline.pipeline.SourceExtractor" + "giant_python.pipeline.pipeline.extract_sources", + return_value=summary, ) as extractor: - extractor.for_scan_mode.return_value.run.return_value = "SUMMARY" result = self.pipe.extract() self.assertIs(result, self.pipe) - self.assertEqual(self.pipe.summary, "SUMMARY") - - def test_run_all_sequences_stages(self): - """run_all sequences organize -> register -> annotate -> extract.""" - with mock.patch.object( - self.pipe, "organize", return_value=self.pipe - ) as organize, mock.patch.object( - self.pipe, "register", return_value=self.pipe - ) as register, mock.patch.object( - self.pipe, "annotate", return_value=self.pipe - ) as annotate, mock.patch.object( - self.pipe, "extract", return_value=self.pipe - ) as extract: - result = self.pipe.run_all("/data", annotate=True) - organize.assert_called_once_with("/data") - register.assert_called_once() - annotate.assert_called_once() - extract.assert_called_once() - self.assertIs(result, self.pipe) - - def test_run_all_skips_annotation(self): - """run_all skips annotation when annotate is False.""" - with mock.patch.object( - self.pipe, "organize", return_value=self.pipe - ), mock.patch.object( - self.pipe, "register", return_value=self.pipe - ), mock.patch.object( - self.pipe, "annotate", return_value=self.pipe - ) as annotate, mock.patch.object( - self.pipe, "extract", return_value=self.pipe - ): - self.pipe.run_all("/data") - annotate.assert_not_called() + self.assertIs(self.pipe.summary, summary) + self.assertIs(extractor.call_args[0][0], self.pipe.tt) def test_from_trial_table(self): - """from_trial_table raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - Pipeline.from_trial_table("trial_table.h5", "slap2") - - def test_from_config(self): - """from_config raises NotImplementedError.""" - with self.assertRaises(NotImplementedError): - Pipeline.from_config("config.yaml") + """from_trial_table loads once and retains the provided filename.""" + table = TrialTable(savedr=Path("/results")) + path = Path("/data/nonstandard_name.h5") + with mock.patch.object( + TrialTable, "from_h5", return_value=table + ) as load: + pipeline = Pipeline.from_trial_table(path) + load.assert_called_once_with(path) + self.assertIs(pipeline.tt, table) + self.assertEqual(pipeline.trial_table_path, path) + self.assertEqual(table.source_path, path) if __name__ == "__main__": diff --git a/tests/test_staging_example.py b/tests/test_staging_example.py new file mode 100644 index 0000000..a95888c --- /dev/null +++ b/tests/test_staging_example.py @@ -0,0 +1,547 @@ +"""Safety and relocation contracts for the standalone staging example.""" + +import contextlib +import io +import os +import subprocess +import sys +import tempfile +import textwrap +import unittest +from dataclasses import asdict +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import call, patch + +from examples import extract_band as example +from giant_python import AnnotationOptions, BandSiloParams, ExecutionOptions +from giant_python.models.trial_table import Slap2Info, TrialTable + +ROOT = Path(__file__).resolve().parents[1] + + +class TestExampleImport(unittest.TestCase): + """Import and help work without loading the scientific backend.""" + + def test_import_has_no_filesystem_environment_or_backend_side_effects( + self, + ): + """Execute an import in a fresh interpreter with mutation guards.""" + code = textwrap.dedent(""" + import argparse + import builtins + import contextlib + import dataclasses + import os + import pathlib + import shutil + import sys + import types + import typing + from unittest.mock import patch + + path = pathlib.Path('examples/extract_band.py') + source = compile(path.read_text(), str(path), 'exec') + module = types.ModuleType('staging_import_check') + sys.modules[module.__name__] = module + before = dict(os.environ) + original_import = builtins.__import__ + forbidden = { + 'giant_python', 'numpy', 'scipy', 'matplotlib', 'cv2', + 'tkinter', 'joblib', 'slap2_utils', 'torch', 'h5py', + } + + def checked_import(name, *args, **kwargs): + assert name.split('.')[0] not in forbidden, name + return original_import(name, *args, **kwargs) + + def audit(event, args): + assert event not in { + 'open', 'os.mkdir', 'os.remove', 'os.rmdir', 'os.rename', + 'os.symlink', 'os.link', 'os.putenv', 'os.unsetenv', + 'os.chdir', 'os.chmod', 'os.truncate', 'os.utime', + 'os.listdir', 'os.scandir', + }, (event, args) + + sys.addaudithook(audit) + with patch('builtins.__import__', side_effect=checked_import): + exec(source, module.__dict__) + assert dict(os.environ) == before + assert not forbidden.intersection(sys.modules) + assert callable(module.main) + """) + result = subprocess.run( + [sys.executable, "-B", "-c", code], + cwd=ROOT, + capture_output=True, + text=True, + timeout=30, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + def test_help_exits_before_validation(self): + """Help requires neither a dataset nor GIAnT imports.""" + with ( + patch.object(example, "prepare_run") as prepare, + contextlib.redirect_stdout(io.StringIO()) as output, + self.assertRaises(SystemExit) as raised, + ): + example.main(["--help"]) + self.assertEqual(raised.exception.code, 0) + self.assertIn("--stage-data", output.getvalue()) + prepare.assert_not_called() + + +class TestRelativeNames(unittest.TestCase): + """Metadata cannot bypass input roots on either path convention.""" + + def test_relative_nested_and_empty_entries_are_accepted(self): + """Preserve relative subdirectories and absent per-trial entries.""" + example.validate_relative_names( + [ + [None, "", "trial.dat"], + ["raw/trial.dat", r"raw\trial.dat", "."], + ], + "filename", + ) + + def test_absolute_drive_unc_and_parent_names_are_rejected(self): + """POSIX and Windows escapes fail regardless of the host OS.""" + for name in ( + "/raw/trial.dat", + "../trial.dat", + "raw/../trial.dat", + r"..\trial.dat", + r"raw\..\trial.dat", + r"raw/..\trial.dat", + r"C:\raw\trial.dat", + "C:/raw/trial.dat", + "C:trial.dat", + r"\raw\trial.dat", + r"\\server\share\trial.dat", + "//server/share/trial.dat", + ): + for field in ("filename", "motion_correction/fn_adata"): + with self.subTest(name=name, field=field): + with self.assertRaisesRegex(ValueError, field): + example.validate_relative_names([["ok", name]], field) + + +class TestStagingExample(unittest.TestCase): + """Use tiny fixture files and a mocked backend, never a real dataset.""" + + def setUp(self): + """Create independent input trees and a portable loaded table.""" + temporary = tempfile.TemporaryDirectory(prefix="giant-example-") + self.addCleanup(temporary.cleanup) + self.root = Path(temporary.name).resolve() + self.data = self.root / "raw" + self.motion = self.root / "registered" + self.annotations = self.root / "rois" + self.scratch = self.root / "scratch" + self.results = self.root / "run" + self.table_path = self.root / "selected-table.h5" + for directory in ( + self.data, + self.motion, + self.annotations, + self.scratch, + ): + directory.mkdir() + self.table_path.write_bytes(b"unchanged table provenance") + (self.data / "trial.dat").write_bytes(b"raw") + (self.data / "trial.meta").write_bytes(b"sidecar") + (self.data / "reference.tif").write_bytes(b"reference") + (self.motion / "bandRegLookupTable.h5").write_bytes(b"lookup") + (self.motion / "alignment.h5").write_bytes(b"alignment") + (self.annotations / "annotations.h5").write_bytes(b"annotations") + self.table = TrialTable( + datadr=Path("old/raw"), + savedr=Path("old/results"), + filename=[["trial.dat"]], + slap2_info=Slap2Info(), + motion_correction={"fn_adata": [["alignment.h5"]]}, + source_path=Path("old/table.h5"), + ) + self.load = self.enterContext( + patch.object(TrialTable, "from_h5", return_value=self.table) + ) + self.extract = self.enterContext( + patch( + "giant_python.extract_band_sources", + return_value=SimpleNamespace(paths=[object()]), + ) + ) + + def argv(self, *extra): + """Build explicit input arguments, allowing final flag overrides.""" + return [ + "--trial-table", + str(self.table_path), + "--data-dir", + str(self.data), + "--motion-correction", + str(self.motion), + "--results-dir", + str(self.results), + *map(str, extra), + ] + + def invoke(self, *extra): + """Run quietly with the mocked extraction service.""" + with contextlib.redirect_stdout(io.StringIO()): + return example.main(self.argv(*extra)) + + def snapshot(self): + """Capture fixture contents, including links without following them.""" + return { + str(path.relative_to(self.root)): ( + ("link", str(path.readlink())) + if path.is_symlink() + else ( + ("directory", None) + if path.is_dir() + else ("file", path.read_bytes()) + ) + ) + for path in self.root.rglob("*") + } + + def assert_rejected(self, error, *extra): + """Rejected inputs must not stage, extract, write or change env.""" + before = self.snapshot() + environment = dict(os.environ) + extraction_calls = list(self.extract.mock_calls) + with ( + patch.object(example, "stage_directory") as stage, + patch.object(example.shutil, "copy2") as copy, + patch.object(Path, "mkdir") as mkdir, + contextlib.redirect_stderr(io.StringIO()), + self.assertRaises(error), + ): + self.invoke("--joblib-temp", self.scratch, *extra) + stage.assert_not_called() + copy.assert_not_called() + mkdir.assert_not_called() + self.assertEqual(self.extract.mock_calls, extraction_calls) + self.assertEqual(self.snapshot(), before) + self.assertEqual(dict(os.environ), environment) + + def test_default_run_relocates_loaded_model_without_rewriting_source(self): + """Default staging changes only the copied model's effective roots.""" + original = asdict(self.table) + self.assertEqual(self.invoke(), 0) + self.load.assert_called_once_with(self.table_path) + self.extract.assert_called_once() + args, kwargs = self.extract.call_args + table, science = args + self.assertIsInstance(table, TrialTable) + self.assertIsNot(table, self.table) + self.assertEqual(table.datadr, self.data) + self.assertEqual(table.savedr, self.results) + self.assertEqual( + table.source_path, self.results / "input_trial_table.h5" + ) + self.assertEqual(asdict(self.table), original) + self.assertEqual(science, BandSiloParams()) + self.assertEqual(science.peakth, 7.0) + self.assertEqual(kwargs["execution"], ExecutionOptions()) + self.assertEqual( + kwargs["annotations"], + AnnotationOptions(False, False, "SLAP2 User"), + ) + self.assertEqual( + table.source_path.read_bytes(), self.table_path.read_bytes() + ) + self.assertEqual( + (self.results / "motion_correction/alignment.h5").read_bytes(), + b"alignment", + ) + self.assertFalse((self.results / "input").exists()) + + def test_copy_stages_sidecars_references_annotations_and_options(self): + """Copy input trees and forward split policy/science overrides.""" + self.invoke( + "--stage-data", + "copy", + "--use-rois", + "--annotations-dir", + self.annotations, + "--max-workers", + 2, + "--max-trials", + 3, + "--verbose", + "--operator", + "Ada", + "--denoise-window-s", + 2, + "--vif", + 1.67, + ) + table, params = self.extract.call_args.args + self.assertEqual(table.datadr, self.results / "input") + for name in ("trial.dat", "trial.meta", "reference.tif"): + self.assertEqual( + (table.datadr / name).read_bytes(), + (self.data / name).read_bytes(), + ) + self.assertEqual( + (self.results / "annotations/annotations.h5").read_bytes(), + b"annotations", + ) + self.assertEqual(params, BandSiloParams(denoise_window_s=2, vif=1.67)) + self.assertEqual( + self.extract.call_args.kwargs, + { + "execution": ExecutionOptions(2, True, 3), + "annotations": AnnotationOptions(True, False, "Ada"), + }, + ) + + def test_symlink_staging_uses_correct_roots_without_link_privileges(self): + """Mock OS link creation while verifying real staging dispatch.""" + with patch.object(Path, "symlink_to", autospec=True) as link: + self.invoke("--stage-data", "symlink", "--stage-motion", "symlink") + self.assertEqual( + link.call_args_list, + [ + call( + self.results / "motion_correction", + self.motion, + target_is_directory=True, + ), + call( + self.results / "input", + self.data, + target_is_directory=True, + ), + ], + ) + table = self.extract.call_args.args[0] + self.assertEqual(table.datadr, self.results / "input") + self.assertEqual(table.savedr, self.results) + + def test_existing_directory_and_file_are_never_replaced(self): + """Existing contents survive even when output is empty or a file.""" + for destination in (self.data, self.table_path, self.scratch): + with self.subTest(destination=destination): + self.assert_rejected( + FileExistsError, "--results-dir", destination + ) + + def test_existing_and_dangling_results_links_are_never_replaced(self): + """Check live/dead link preflight without requiring link privilege.""" + exists, is_symlink = Path.exists, Path.is_symlink + for name, live in (("live", True), ("dead", False)): + link = self.root / name + with ( + self.subTest(name=name), + patch.object( + Path, + "exists", + autospec=True, + side_effect=lambda p: live if p == link else exists(p), + ), + patch.object( + Path, + "is_symlink", + autospec=True, + side_effect=lambda p: p == link or is_symlink(p), + ), + patch.object(Path, "unlink", autospec=True) as unlink, + ): + self.assert_rejected(FileExistsError, "--results-dir", link) + unlink.assert_not_called() + + def test_dangling_link_guard_without_os_link_privileges(self): + """The link predicate is checked even when exists() is false.""" + with ( + patch.object(Path, "exists", return_value=False), + patch.object(Path, "is_symlink", return_value=True), + self.assertRaises(FileExistsError), + ): + example.validate_destination(self.results, (self.data,)) + + def test_output_nested_in_any_input_is_rejected(self): + """Reject output within raw, motion, or annotation input roots.""" + for source in (self.data, self.motion, self.annotations): + with self.subTest(source=source): + self.assert_rejected( + ValueError, + "--results-dir", + source / "new/run", + "--annotations-dir", + self.annotations, + ) + + def test_missing_and_wrong_kind_paths_fail_before_staging(self): + """Check input paths and optional scratch roots before staging.""" + for flag, path, error in ( + ("--trial-table", self.root / "missing.h5", FileNotFoundError), + ("--trial-table", self.data, FileNotFoundError), + ("--data-dir", self.table_path, NotADirectoryError), + ("--data-dir", self.root / "absent", FileNotFoundError), + ("--motion-correction", self.scratch, FileNotFoundError), + ("--annotations-dir", self.scratch, FileNotFoundError), + ("--joblib-temp", self.table_path, NotADirectoryError), + ("--joblib-temp", self.root / "absent", FileNotFoundError), + ): + with self.subTest(flag=flag, path=path): + self.assert_rejected(error, flag, path) + self.load.assert_not_called() + + def test_missing_registration_metadata_fails_before_staging(self): + """A loaded table must identify SLAP2 data and alignment filenames.""" + for slap2, motion in ( + (None, {"fn_adata": [["alignment.h5"]]}), + (Slap2Info(), None), + (Slap2Info(), {}), + (Slap2Info(), {"alignHz": 80}), + ): + with self.subTest(slap2=slap2, motion=motion): + self.table.slap2_info = slap2 + self.table.motion_correction = motion + self.assert_rejected(ValueError) + + def test_unsafe_loaded_names_fail_before_staging(self): + """Both metadata fields are validated on the loaded table.""" + for field in ("filename", "fn_adata"): + with self.subTest(field=field): + self.table.filename = [["trial.dat"]] + self.table.motion_correction = {"fn_adata": [["alignment.h5"]]} + if field == "filename": + self.table.filename = [["../trial.dat"]] + else: + self.table.motion_correction[field] = [ + [r"..\alignment.h5"] + ] + self.assert_rejected(ValueError) + + def test_bad_science_and_execution_options_cannot_mutate_run(self): + """Validate finite positive settings before mkdir/copy/env changes.""" + for flag in ( + "--analyze-hz", + "--decay-tau-s", + "--baseline-window-s", + "--denoise-window-s", + "--vif", + "--d-xy", + "--peak-buffer", + "--psf-dilation", + "--max-workers", + "--max-trials", + ): + with self.subTest(flag=flag): + self.assert_rejected(ValueError, flag, 0) + for value in ("nan", "inf", "-1"): + with self.subTest(value=value): + self.assert_rejected(ValueError, "--analyze-hz", value) + + def test_invalid_roi_policy_fails_before_reading_table(self): + """Drawing requires ROI use; headless ROI use needs an input file.""" + for flags in (("--interactive",), ("--use-rois",)): + with self.subTest(flags=flags): + self.assert_rejected(SystemExit, *flags) + self.load.assert_not_called() + + def test_joblib_override_is_visible_only_during_execution(self): + """Restore both absent and pre-existing environment values.""" + for previous in (None, "", "original-scratch"): + for fails in (False, True): + with self.subTest(previous=previous, fails=fails): + destination = self.root / f"run-{previous}-{fails}" + with patch.dict(os.environ): + if previous is None: + os.environ.pop("JOBLIB_TEMP_FOLDER", None) + else: + os.environ["JOBLIB_TEMP_FOLDER"] = previous + environment = dict(os.environ) + + def extract(*args, **kwargs): + """Observe the worker setting in the backend.""" + self.assertEqual( + os.environ["JOBLIB_TEMP_FOLDER"], + str(self.scratch), + ) + if fails: + raise RuntimeError("backend failed") + return SimpleNamespace(paths=[]) + + self.extract.side_effect = extract + flags = ( + "--results-dir", + destination, + "--joblib-temp", + self.scratch, + ) + if fails: + with self.assertRaisesRegex( + RuntimeError, "backend failed" + ): + self.invoke(*flags) + else: + self.invoke(*flags) + self.assertEqual(dict(os.environ), environment) + self.assertTrue( + (destination / "input_trial_table.h5").is_file() + ) + self.assert_rejected( + FileExistsError, "--results-dir", destination + ) + + def test_no_joblib_flag_preserves_environment_during_extraction(self): + """Extraction observes the caller's environment without overrides.""" + environment = dict(os.environ) + + def extract(*args, **kwargs): + """Check the environment at the point workers would be launched.""" + self.assertEqual(dict(os.environ), environment) + return SimpleNamespace(paths=[]) + + self.extract.side_effect = extract + self.invoke() + self.assertEqual(dict(os.environ), environment) + + def test_destination_created_after_validation_is_not_overwritten(self): + """Exclusive mkdir refuses an output that appeared after preflight.""" + args = example.build_parser().parse_args(self.argv()) + run = example.prepare_run(args) + self.results.mkdir() + marker = self.results / "keep.txt" + marker.write_bytes(b"keep") + with ( + patch.object(example.shutil, "copy2") as copy, + self.assertRaises(FileExistsError), + ): + example.stage_and_extract(run, args) + copy.assert_not_called() + self.extract.assert_not_called() + self.assertEqual(marker.read_bytes(), b"keep") + + def test_stage_directory_guards_existing_destinations_and_invalid_mode( + self, + ): + """Staging never invokes copy/link on an existing or invalid target.""" + with ( + patch.object(example.shutil, "copytree") as copy, + patch.object(Path, "symlink_to") as link, + ): + for mode in ("copy", "symlink"): + with self.subTest(mode=mode): + with self.assertRaises(FileExistsError): + example.stage_directory(self.data, self.scratch, mode) + with ( + patch.object(Path, "exists", return_value=False), + patch.object(Path, "is_symlink", return_value=True), + self.assertRaises(FileExistsError), + ): + example.stage_directory(self.data, self.results, mode) + with self.assertRaisesRegex(ValueError, "Unknown staging mode"): + example.stage_directory(self.data, self.results, "invalid") + copy.assert_not_called() + link.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_workflows.py b/tests/test_workflows.py new file mode 100644 index 0000000..51a3567 --- /dev/null +++ b/tests/test_workflows.py @@ -0,0 +1,320 @@ +"""Mocked service contracts: path identity, dispatch and API convergence.""" + +import sys +import unittest +from copy import deepcopy +from importlib import import_module +from pathlib import Path +from types import ModuleType +from unittest import mock + +import giant_python +from giant_python import cli +from giant_python.models import ( + AnnotationOptions, + BandSiloParams, + ExecutionOptions, + ExperimentSummary, + TrialTable, +) +from giant_python.pipeline import ( + Pipeline, + annotate_rois, + extract_band_sources, + extract_sources, +) + + +class TestPublicExports(unittest.TestCase): + """Lazy exports resolve to canonical objects and advertise real APIs.""" + + def test_lazy_exports_match_their_owner(self): + """Every advertised model and workflow resolves and is cached.""" + for name in giant_python.__all__: + with self.subTest(name=name): + owner = ( + "giant_python.models" + if name in giant_python._MODEL_EXPORTS + else "giant_python.pipeline" + ) + value = giant_python.__getattr__(name) + self.assertIs(value, getattr(import_module(owner), name)) + self.assertIs(giant_python.__dict__[name], value) + self.assertIn(name, dir(giant_python)) + + def test_unknown_export_has_standard_attribute_error(self): + """Unknown names raise AttributeError without importing a backend.""" + with self.assertRaisesRegex(AttributeError, "no attribute 'missing'"): + giant_python.__getattr__("missing") + + def test_legacy_apis_are_absent(self): + """Removed carriers and adapter classes have no reexport shims.""" + names = { + "giant_python.models.params": ( + "SiloParams", + "set_params", + "to_legacy_silo_params", + ), + "giant_python.models": ( + "SiloParams", + "set_params", + "to_legacy_silo_params", + ), + "giant_python.pipeline.extract": ( + "SourceExtractor", + "BandSourceExtractor", + "StandardSourceExtractor", + ), + "giant_python.pipeline": ( + "SourceExtractor", + "BandSourceExtractor", + "StandardSourceExtractor", + ), + "giant_python": ( + "SiloParams", + "SourceExtractor", + "BandSourceExtractor", + "StandardSourceExtractor", + ), + } + for module_name, removed in names.items(): + module = import_module(module_name) + for name in removed: + with self.subTest(module=module_name, name=name): + self.assertFalse(hasattr(module, name)) + + +class TestSharedServices(unittest.TestCase): + """All supported entry points converge without importing real backends.""" + + def setUp(self): + """Inject only the backend modules; keep public delegation real.""" + self.summary = ExperimentSummary() + self.extract = mock.Mock(return_value=self.summary) + self.annotate = mock.Mock(return_value="annotations.h5") + workflow = ModuleType("giant_python.extraction.band.workflow") + workflow.extract_band_sources = self.extract + annotation = ModuleType("giant_python.extraction.band.annotation") + annotation.annotate_band_rois = self.annotate + patch = mock.patch.dict( + sys.modules, + { + workflow.__name__: workflow, + annotation.__name__: annotation, + }, + ) + patch.start() + self.addCleanup(patch.stop) + + def test_direct_function_forwards_original_inputs(self): + """Scientific/run options are resolved once by the backend service.""" + table = TrialTable() + params = BandSiloParams() + execution = ExecutionOptions(max_workers=2) + annotations = AnnotationOptions(interactive=False) + result = extract_band_sources( + table, params, execution=execution, annotations=annotations + ) + self.assertIs(result, self.summary) + args, kwargs = self.extract.call_args + self.assertIs(args[0], table) + self.assertIs(args[1], params) + self.assertIs(kwargs["execution"], execution) + self.assertIs(kwargs["annotations"], annotations) + + def test_only_canonical_params_keyword(self): + """All public services accept params, not params_in.""" + for function in (extract_band_sources, extract_sources, annotate_rois): + with self.subTest(function=function.__name__): + function("custom.h5", params=BandSiloParams()) + with self.assertRaises(TypeError): + function("custom.h5", params_in={}) + + def test_generic_dispatch(self): + """No params, band params and explicit band all select band.""" + for params, mode in ( + (None, None), + (BandSiloParams(), None), + ({}, None), + (BandSiloParams(), "band"), + ): + with self.subTest(params=params, mode=mode): + self.assertIs( + extract_sources("custom.h5", params, scan_mode=mode), + self.summary, + ) + self.assertEqual(self.extract.call_count, 4) + + def test_explicit_unsupported_modes_are_never_ignored(self): + """Explicit routing is checked before backend invocation.""" + cases = ( + {"params": BandSiloParams(), "scan_mode": "standard"}, + {"scan_mode": "unknown"}, + {"microscope": "bergamo", "scan_mode": "band"}, + ) + for kwargs in cases: + with self.subTest(kwargs=kwargs): + with self.assertRaisesRegex(ValueError, "supported"): + extract_sources("custom.h5", **kwargs) + with self.assertRaisesRegex(ValueError, "supported"): + annotate_rois("custom.h5", **kwargs) + self.extract.assert_not_called() + self.annotate.assert_not_called() + + def test_missing_input_is_actionable(self): + """An omitted table fails before a heavy backend is invoked.""" + for function in (extract_band_sources, extract_sources, annotate_rois): + with self.subTest(function=function.__name__): + with self.assertRaisesRegex(ValueError, "trial-table"): + function(None) + self.extract.assert_not_called() + self.annotate.assert_not_called() + + def test_annotation_forwards_options_without_mutation(self): + """Annotation forwards split options directly to the backend.""" + table = TrialTable() + params = BandSiloParams() + options = AnnotationOptions(interactive=False, operator="Ada") + before = deepcopy((params, options)) + result = annotate_rois(table, params, annotations=options) + self.assertEqual(result, "annotations.h5") + input, science = self.annotate.call_args.args + self.assertIs(input, table) + self.assertIs(science, params) + self.assertIs(self.annotate.call_args.kwargs["annotations"], options) + self.assertIsNone(self.annotate.call_args.kwargs["execution"]) + self.assertEqual((params, options), before) + + def test_pipeline_loads_nonstandard_filename_only_once(self): + """A table outside savedr is preserved through annotate/extract.""" + table = TrialTable(savedr=Path("/different/results")) + path = Path("/metadata/session_curated_v2.h5") + with mock.patch.object( + TrialTable, "from_h5", return_value=table + ) as load: + pipeline = Pipeline.from_trial_table(path) + result = pipeline.annotate().extract() + load.assert_called_once_with(path) + self.assertIs(result, pipeline) + self.assertIs(pipeline.summary, self.summary) + self.assertEqual(pipeline.trial_table_path, path) + self.assertEqual(table.source_path, path) + self.assertIs(self.annotate.call_args.args[0], table) + self.assertIs(self.extract.call_args.args[0], table) + self.assertTrue(self.extract.call_args.kwargs["annotations"].enabled) + + def test_in_memory_pipeline_never_guesses_source_path(self): + """savedr does not imply a trial_table.h5 exists in that directory.""" + for savedr in (None, Path("/results")): + with self.subTest(savedr=savedr): + table = TrialTable(savedr=savedr) + with mock.patch.object(TrialTable, "from_h5") as load: + pipeline = Pipeline.from_trial_table(table) + pipeline.extract() + load.assert_not_called() + self.assertIs(pipeline.tt, table) + self.assertIsNone(pipeline.trial_table_path) + self.assertIsNone(table.source_path) + self.assertIs(self.extract.call_args.args[0], table) + + def test_in_memory_known_source_path_is_retained(self): + """Metadata already loaded by IO retains its actual source path.""" + path = Path("/metadata/curated.h5") + table = TrialTable(source_path=path) + pipeline = Pipeline.from_trial_table(table) + self.assertIs(pipeline.tt, table) + self.assertEqual(pipeline.trial_table_path, path) + self.assertEqual(table.source_path, path) + + def test_annotation_chain_copies_policy(self): + """Explicit annotation enables later ROI use, not caller options.""" + options = AnnotationOptions(interactive=False) + pipeline = Pipeline.from_trial_table(TrialTable(), annotations=options) + pipeline.annotate().extract() + self.assertFalse(options.enabled) + self.assertTrue(pipeline.annotations.enabled) + self.assertIsNot(pipeline.annotations, options) + + def test_top_level_and_cli_share_backend_service(self): + """Direct, top-level, Pipeline and CLI paths reach the same driver.""" + self.assertIs(giant_python.extract_sources, extract_sources) + self.assertIs(giant_python.annotate_rois, annotate_rois) + result = giant_python.extract_sources("curated.h5") + self.assertIs(result, self.summary) + self.assertEqual( + cli.main( + [ + "extract", + "curated.h5", + "--headless", + "--max-workers", + "2", + ] + ), + 0, + ) + args, kwargs = self.extract.call_args + self.assertEqual(args[0], "curated.h5") + self.assertIsInstance(args[1], BandSiloParams) + self.assertEqual(kwargs["execution"].max_workers, 2) + self.assertFalse(kwargs["annotations"].interactive) + self.assertEqual(cli.main(["annotate", "curated.h5", "--headless"]), 0) + self.assertEqual(self.annotate.call_args.args[0], "curated.h5") + self.assertFalse( + self.annotate.call_args.kwargs["annotations"].interactive + ) + + +class TestPipelineErrors(unittest.TestCase): + """Unsupported or uninitialized pipeline use fails before side effects.""" + + def test_mixed_science_rejected_before_loading(self): + """Real services reject policy/routing fields rather than dispatch.""" + from giant_python.extraction.band import annotation, workflow + + with ( + mock.patch.object(workflow, "load_trial_table") as extract_load, + mock.patch.object(annotation, "load_trial_table") as roi_load, + ): + for field, value in ( + ("max_workers", 2), + ("operator", "Ada"), + ("draw_user_rois", True), + ("scan_mode", "band"), + ("microscope", "slap2"), + ): + params = {"peakth": 7.0, field: value} + for function in ( + extract_band_sources, + extract_sources, + annotate_rois, + ): + with self.subTest(field=field, api=function.__name__): + with self.assertRaisesRegex(TypeError, field): + function("curated.h5", params=params) + pipeline = Pipeline.from_trial_table( + TrialTable(), params=params + ) + for stage in (pipeline.extract, pipeline.annotate): + with self.subTest(field=field, stage=stage.__name__): + with self.assertRaisesRegex(TypeError, field): + stage() + extract_load.assert_not_called() + roi_load.assert_not_called() + + def test_missing_table(self): + """No table is a clear error, not an AttributeError on savedr.""" + pipeline = Pipeline() + for name in ("annotate", "extract"): + with self.subTest(stage=name): + with self.assertRaisesRegex(ValueError, "No trial table"): + getattr(pipeline, name)() + + def test_invalid_table_input(self): + """Invalid input types cannot be mistaken for paths.""" + with self.assertRaisesRegex(TypeError, "TrialTable"): + Pipeline.from_trial_table(None) + + +if __name__ == "__main__": + unittest.main() From b46039ac05cbc2edcdf9f55c522a3298e6bf4e0a Mon Sep 17 00:00:00 2001 From: Michael Xie Date: Mon, 21 Sep 2026 11:29:40 -0700 Subject: [PATCH 36/55] fix background_interpolation to always be cubic --- src/giant_python/extraction/band/workflow.py | 4 +-- src/giant_python/models/params.py | 8 +----- tests/test_bandsilo_pipeline.py | 22 +++++++++++++--- tests/test_configuration.py | 27 ++++++++++++-------- tests/test_models.py | 4 +-- 5 files changed, 40 insertions(+), 25 deletions(-) diff --git a/src/giant_python/extraction/band/workflow.py b/src/giant_python/extraction/band/workflow.py index 873a332..5af245b 100644 --- a/src/giant_python/extraction/band/workflow.py +++ b/src/giant_python/extraction/band/workflow.py @@ -158,7 +158,7 @@ def _params_dict( "analyzeHz": params.analyze_hz, "denoiseWindow_s": params.denoise_window_s, "baselineWindow_s": params.baseline_window_s, - "backgroundInterpolation": params.background_interpolation, + "backgroundInterpolation": "cubic", "decayTau_s": params.decay_tau_s, "dXY": params.d_xy, "sparse_fac": params.sparse_fac, @@ -436,7 +436,7 @@ def _estimate_background( sel_2d, umyx, mot_inds_yx, - method=params.background_interpolation, + method="cubic", ) baseline_window = bg.baseline_window_frames( align_hz, params.baseline_window_s diff --git a/src/giant_python/models/params.py b/src/giant_python/models/params.py index 1665514..cb10ebc 100644 --- a/src/giant_python/models/params.py +++ b/src/giant_python/models/params.py @@ -39,8 +39,6 @@ class BandSiloParams: Calcium-decay time constant in seconds. baseline_window_s, denoise_window_s : float Baseline and temporal denoising windows in seconds. - background_interpolation : {'cubic', 'linear'} - Sparse-superpixel background interpolation method. vif : float Noise-model variance inflation factor. d_xy : int @@ -65,12 +63,12 @@ class BandSiloParams: ----- Scheduling, trial limits, logging and annotation policy are deliberately separate from science. Numerical defaults preserve the band algorithm. + Sparse-superpixel background interpolation always uses cubic splines. """ analyze_hz: float = 100.0 decay_tau_s: float = 0.15 baseline_window_s: float = 4.0 - background_interpolation: str = "cubic" denoise_window_s: float = 1.0 vif: float = 1.38 d_xy: int = 5 @@ -97,10 +95,6 @@ def __post_init__(self) -> None: _positive(name, getattr(self, name), integer=True) if self.num_channels is not None: _positive("num_channels", self.num_channels, integer=True) - if self.background_interpolation not in ("cubic", "linear"): - raise ValueError( - "background_interpolation must be 'cubic' or 'linear'" - ) if ( isinstance(self.activity_channel, bool) or not isinstance(self.activity_channel, Integral) diff --git a/tests/test_bandsilo_pipeline.py b/tests/test_bandsilo_pipeline.py index 83f8f7a..643d314 100644 --- a/tests/test_bandsilo_pipeline.py +++ b/tests/test_bandsilo_pipeline.py @@ -16,13 +16,20 @@ from giant_python.extraction.band import result_assembly as pl from giant_python.extraction.band import workflow as wf -from giant_python.extraction.band.types import TrialTraceResult +from giant_python.extraction.band.types import ( + ResolvedAcquisition, + TrialTraceResult, +) from giant_python.io.experiment_summary import write_summary from giant_python.models.experiment_summary import ( FRAME_INFO_KEYS, ExperimentSummary, ) -from giant_python.models.params import BandSiloParams, ExecutionOptions +from giant_python.models.params import ( + AnnotationOptions, + BandSiloParams, + ExecutionOptions, +) def _trial(n_frames, n_sources, n_channels, n_soma): @@ -265,6 +272,15 @@ def test_dominant_retained_motion_shifts_only_median_geometry(self): class TestStageComposition(unittest.TestCase): """Check background wiring and empty localization without sparse fits.""" + def test_persisted_background_interpolation_is_cubic(self): + """Output metadata records the fixed background algorithm.""" + metadata = wf._params_dict( + BandSiloParams(), + ResolvedAcquisition(num_channels=1, align_hz=100.0), + AnnotationOptions(), + ) + self.assertEqual(metadata["backgroundInterpolation"], "cubic") + def test_default_execution_skips_empty_planes_and_preserves_grid(self): """Interpolate selected planes; baseline and assembly share data.""" data = np.array([[3.0, 4.0], [5.0, 6.0]], dtype=np.float32) @@ -336,7 +352,7 @@ def test_default_execution_skips_empty_planes_and_preserves_grid(self): self.assertIs(interpolate.call_args.args[3], plane_pixels) self.assertEqual( interpolate.call_args.kwargs["method"], - params.background_interpolation, + "cubic", ) rolling.assert_called_once() np.testing.assert_array_equal(rolling.call_args.args[0], interpolated) diff --git a/tests/test_configuration.py b/tests/test_configuration.py index bdbd175..10d344c 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -109,17 +109,22 @@ def test_positive_counts(self): self.assertIsNone(BandSiloParams().num_channels) self.assertEqual(BandSiloParams(num_channels=2).num_channels, 2) - def test_interpolation_and_activity_channel(self): - """Only existing interpolation modes and activity channel zero work.""" - for mode in ("linear", "cubic"): - self.assertEqual( - BandSiloParams( - background_interpolation=mode - ).background_interpolation, - mode, - ) - with self.assertRaisesRegex(ValueError, "interpolation"): - BandSiloParams(background_interpolation="nearest") + def test_background_interpolation_is_not_configurable(self): + """Neither typed nor dictionary options accept interpolation modes.""" + for mode in ("linear", "cubic", "nearest"): + with self.subTest(mode=mode): + options = {"background_interpolation": mode} + with self.assertRaisesRegex( + TypeError, "background_interpolation" + ): + BandSiloParams(**options) + with self.assertRaisesRegex( + TypeError, "background_interpolation" + ): + resolve_band_options(options) + + def test_activity_channel(self): + """Only activity channel zero is supported.""" for channel in (1, -1, 0.0, False, None): with self.subTest(channel=channel): with self.assertRaisesRegex(ValueError, "activity_channel"): diff --git a/tests/test_models.py b/tests/test_models.py index e4d4f28..ea85854 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -61,8 +61,8 @@ def test_source_and_user_roi(self): self.assertEqual(UserRoi().type, "polygon") def test_param_models(self): - """Param models expose typed defaults.""" - self.assertEqual(BandSiloParams().background_interpolation, "cubic") + """Fixed interpolation is not exposed as a model parameter.""" + self.assertFalse(hasattr(BandSiloParams(), "background_interpolation")) def test_band_value_params_have_concrete_defaults(self): """Value params default to concrete (non-None) values.""" From 66b3a39be45785a3378619192c73460156c2648b Mon Sep 17 00:00:00 2001 From: Michael Xie Date: Mon, 21 Sep 2026 11:38:02 -0700 Subject: [PATCH 37/55] remove num_channels and activity_channel parameters --- README.md | 4 +-- docs/source/migration.rst | 9 ++++-- src/giant_python/extraction/band/workflow.py | 9 ++---- src/giant_python/models/params.py | 19 ++----------- tests/test_band_workflow.py | 6 ++-- tests/test_configuration.py | 30 ++++++++++++-------- tests/test_models.py | 14 ++++++--- 7 files changed, 45 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index 010bc8d..b906b1e 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ The Python API preceding this cleanup was unreleased and unused. Compatibility-o - `extract_band_sources(input, params=None, *, execution=None, annotations=None)` accepts an exact filename or loaded `TrialTable`. Use `Pipeline(..., params=...)` or `Pipeline.from_trial_table(..., params=...).extract()` for a session. - `resolve_band_options(params=None, execution=None, annotations=None)` returns fresh validated `(BandSiloParams, ExecutionOptions, AnnotationOptions)` objects. Each argument accepts **only its corresponding dataclass, a dictionary of fields belonging to that scope, or `None`**. Mixed dictionaries, cross-scope fields and unknown keys are rejected; they are not redistributed or converted. Caller inputs are not mutated. -- `params` holds science and optional `num_channels`; `execution` holds `max_workers`, `verbose`, and `max_trials`; `annotations` holds `enabled`, `interactive`, and `operator`. Generic services and `Pipeline` take `microscope` / `scan_mode` as explicit routing options; only SLAP2 band extraction is implemented (`scan_mode=None` also selects band). There is no registration configuration class. +- `params` holds science only; channel count is inferred from acquisition metadata and source activity extraction uses channel zero. `execution` holds `max_workers`, `verbose`, and `max_trials`; `annotations` holds `enabled`, `interactive`, and `operator`. Generic services and `Pipeline` take `microscope` / `scan_mode` as explicit routing options; only SLAP2 band extraction is implemented (`scan_mode=None` also selects band). There is no registration configuration class. - All per-path results are accessed through `ExperimentSummary.paths`. A `PathSummary` owns `sources`, `user_rois`, `visualizations`, `frame_info`, `global_f`, and `z_depths`. There are no flat or aggregate summary properties. Access frame fields as attributes, such as `path.frame_info.offlineXshifts`, or call the shallow `FrameInfo.to_dict()`; `FrameInfo` is not a mapping. - `giant_python.io.read_summary(path)` returns an `ExperimentSummary`; `giant_python.io.write_summary(summary, path)` accepts that model and returns `None`. Their owner is `giant_python.io.experiment_summary`; the model's `from_h5()` / `to_h5()` delegate there. No public assembled-dictionary writer or alternate summary IO alias is supported. - `giant_python.extraction.band.result_assembly.assemble_path_summary()` is the band result assembly boundary and returns a `PathSummary`. Scientific backend code belongs in `giant_python.extraction.band`; shared kernels belong in `giant_python.numerics`, not the removed compatibility namespaces. @@ -154,7 +154,7 @@ These conventions describe the supported subset and the reference fields below: - **Images** are stored `rows x cols` (row = Y, col = X). Motion shifts: `motionDSr` / `motionR` are **row** shifts, `motionDSc` / `motionC` are **column** shifts. - `params/activityChannel` is **1-indexed** into the recording's `numChannels` channels. -The last field is a reference parameter: do not assume every reference parameter is emitted by the band writer. The Python `BandSiloParams.activity_channel` is zero-based; currently only `0` is supported, and source activity arrays for other channels retain NaNs. +The last field is a reference parameter: do not assume every reference parameter is emitted by the band writer. Python source activity extraction always uses channel zero (zero-based), and source activity arrays for other channels retain NaNs. Channel count is inferred from acquisition metadata, not supplied in `BandSiloParams`. **Motion-sign convention:** public `FrameInfo.offlineXshifts`, `offlineYshifts`, and `offlineZshifts` contain alignment displacement with the standard MATLAB SILo sign. Private band reference offsets have the opposite sign and are converted once during `assemble_path_summary()`, not by generic IO. If an experimental file from an unreleased Python checkout exists, establish its producer/alignment provenance before interpreting or changing its shifts; there is no known production Python migration to perform. Never negate MATLAB-produced or already-correct summaries again. Online shifts are unchanged. Matching this sign does not change the existing linear/clamped interpolation to MATLAB's PCHIP/extrapolation. Removing Python compatibility code leaves the HDF5 field spelling `Z_depths`, layout flags, coordinate conversions, scientific defaults, motion-bin ordering, and numerical call order unchanged; no new caching or execution policy is introduced. diff --git a/docs/source/migration.rst b/docs/source/migration.rst index be28bcb..d20b8f2 100644 --- a/docs/source/migration.rst +++ b/docs/source/migration.rst @@ -43,7 +43,7 @@ drawing; missing required annotations fail explicitly. - Scope * - ``params`` - ``BandSiloParams`` - - Band science and optional ``num_channels`` override + - Band science only * - ``execution`` - ``ExecutionOptions`` - ``max_workers``, ``verbose``, ``max_trials`` @@ -63,7 +63,12 @@ Defaults remain peak threshold 7, analysis rate 100 Hz, decay 0.15 seconds, baseline window 4 seconds, denoise window 1 second, VIF 1.38 and PSF dilation 17. Execution defaults to six workers, no trial limit and verbosity off. Annotation defaults to disabled with automatic interactivity and operator ``SLAP2 User``. -``BandSiloParams.activity_channel`` is zero-based and only ``0`` is implemented. +``num_channels`` and ``activity_channel`` have been removed from +``BandSiloParams``. Channel count is inferred from acquisition metadata and +retained in output metadata; source activity extraction always uses channel +zero (zero-based). Supplying either removed option raises ``TypeError``. +If channel count cannot be inferred, extraction fails with ``ValueError``; +provide acquisition metadata for a kept trial rather than a manual override. ``AlignParams`` has been removed; registration has no Python configuration API. Canonical owner map diff --git a/src/giant_python/extraction/band/workflow.py b/src/giant_python/extraction/band/workflow.py index 5af245b..036d626 100644 --- a/src/giant_python/extraction/band/workflow.py +++ b/src/giant_python/extraction/band/workflow.py @@ -99,20 +99,15 @@ def extract_band_sources( trial_table["keep_trials"] = compute_keep_trials( trial_table["fn_adata"], trial_table["filename"], trial_table["datadr"] ) - align_hz, inferred_channels = read_align_info( + align_hz, num_channels = read_align_info( trial_table["fn_adata"], trial_table["keep_trials"], trial_table["n_dmds"], ) - num_channels = ( - params.num_channels - if params.num_channels is not None - else inferred_channels - ) if num_channels is None: raise ValueError( "num_channels could not be determined from the alignment data " - "(no kept trials?); set BandSiloParams.num_channels explicitly." + "(no kept trials?); provide acquisition metadata for a kept trial." ) acquisition = ResolvedAcquisition(num_channels, align_hz) user_rois = _resolve_session_user_rois(result_dr, trial_table, lookup, roi) diff --git a/src/giant_python/models/params.py b/src/giant_python/models/params.py index cb10ebc..cd490ca 100644 --- a/src/giant_python/models/params.py +++ b/src/giant_python/models/params.py @@ -1,6 +1,6 @@ """Scientific configuration and independent execution and annotation policy. -``BandSiloParams`` contains only band science and channel overrides. +``BandSiloParams`` contains only band science. ``ExecutionOptions`` controls scheduling/debug limits; ``AnnotationOptions`` controls ROI use and GUI policy. Resolution copies inputs without retuning the existing algorithm. @@ -53,17 +53,14 @@ class BandSiloParams: psf_dilation : int Positive dilation size selecting the bundled PSF template. Template availability is checked by the backend when loading the PSF. - num_channels : int or None - Positive channel-count override; otherwise infer from acquisition - metadata without modifying this object. - activity_channel : int - Zero-based activity channel. Only channel zero is implemented. Notes ----- Scheduling, trial limits, logging and annotation policy are deliberately separate from science. Numerical defaults preserve the band algorithm. Sparse-superpixel background interpolation always uses cubic splines. + Channel count is inferred from acquisition metadata; source activity + extraction always uses channel zero. """ analyze_hz: float = 100.0 @@ -76,8 +73,6 @@ class BandSiloParams: peakth: float = 7.0 peak_buffer: int = 3 psf_dilation: int = 17 - num_channels: Optional[int] = None - activity_channel: int = 0 def __post_init__(self) -> None: """Validate values supported by the existing numerical path.""" @@ -93,14 +88,6 @@ def __post_init__(self) -> None: _positive(name, getattr(self, name)) for name in ("d_xy", "peak_buffer", "psf_dilation"): _positive(name, getattr(self, name), integer=True) - if self.num_channels is not None: - _positive("num_channels", self.num_channels, integer=True) - if ( - isinstance(self.activity_channel, bool) - or not isinstance(self.activity_channel, Integral) - or self.activity_channel != 0 - ): - raise ValueError("Only activity_channel=0 is implemented") @dataclass diff --git a/tests/test_band_workflow.py b/tests/test_band_workflow.py index 9e8241b..a43490f 100644 --- a/tests/test_band_workflow.py +++ b/tests/test_band_workflow.py @@ -377,7 +377,7 @@ def read_trial( self.assertEqual( before, (asdict(params), asdict(options), asdict(annotation)) ) - self.assertIsNone(params.num_channels) + self.assertFalse(hasattr(params, "num_channels")) def test_exact_nonstandard_selected_trial_table_path(self): """Preserve the chosen filename through extraction and persistence.""" @@ -417,13 +417,13 @@ def test_missing_channel_inference_fails_before_processing(self): process = stack.enter_context(patch.object(wf, "_process_dmd")) writer = stack.enter_context(patch.object(wf, "write_summary")) with self.assertRaisesRegex( - ValueError, "num_channels.*determined" + ValueError, "num_channels.*provide acquisition metadata" ): wf.extract_band_sources("selected.h5", params) rois.assert_not_called() process.assert_not_called() writer.assert_not_called() - self.assertIsNone(params.num_channels) + self.assertFalse(hasattr(params, "num_channels")) def test_enabled_empty_annotations_keep_resolution_context(self): """An enabled empty selection retains geometry and headless policy.""" diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 10d344c..ef9ac08 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -100,14 +100,12 @@ def test_positive_scientific_values(self): BandSiloParams(**{name: value}) def test_positive_counts(self): - """Count overrides require positive integers, not floats or bools.""" - for name in ("d_xy", "peak_buffer", "psf_dilation", "num_channels"): + """Spatial counts require positive integers, not floats or bools.""" + for name in ("d_xy", "peak_buffer", "psf_dilation"): for value in (0, -1, 1.5, True): with self.subTest(name=name, value=value): with self.assertRaisesRegex(ValueError, name): BandSiloParams(**{name: value}) - self.assertIsNone(BandSiloParams().num_channels) - self.assertEqual(BandSiloParams(num_channels=2).num_channels, 2) def test_background_interpolation_is_not_configurable(self): """Neither typed nor dictionary options accept interpolation modes.""" @@ -123,12 +121,20 @@ def test_background_interpolation_is_not_configurable(self): ): resolve_band_options(options) - def test_activity_channel(self): - """Only activity channel zero is supported.""" - for channel in (1, -1, 0.0, False, None): - with self.subTest(channel=channel): - with self.assertRaisesRegex(ValueError, "activity_channel"): - BandSiloParams(activity_channel=channel) + def test_channel_parameters_are_not_configurable(self): + """Removed channel options fail for both typed and dictionary inputs.""" + for name, value in ( + ("num_channels", None), + ("num_channels", 2), + ("activity_channel", 0), + ("activity_channel", 1), + ): + with self.subTest(name=name, value=value): + options = {name: value} + with self.assertRaisesRegex(TypeError, name): + BandSiloParams(**options) + with self.assertRaisesRegex(TypeError, name): + resolve_band_options(options) def test_execution_and_annotation_policy(self): """Execution counts and tri-state GUI settings are validated.""" @@ -151,8 +157,8 @@ def test_execution_and_annotation_policy(self): def test_resolver_revalidates_mutated_carriers(self): """Dataclass mutability cannot bypass validation at resolution.""" science = BandSiloParams() - science.num_channels = 0 - with self.assertRaisesRegex(ValueError, "num_channels"): + science.d_xy = 0 + with self.assertRaisesRegex(ValueError, "d_xy"): resolve_band_options(science) execution = ExecutionOptions() execution.max_workers = 0 diff --git a/tests/test_models.py b/tests/test_models.py index ea85854..cfc9ce4 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -61,8 +61,15 @@ def test_source_and_user_roi(self): self.assertEqual(UserRoi().type, "polygon") def test_param_models(self): - """Fixed interpolation is not exposed as a model parameter.""" - self.assertFalse(hasattr(BandSiloParams(), "background_interpolation")) + """Fixed algorithms and acquisition metadata are not parameters.""" + params = BandSiloParams() + for name in ( + "background_interpolation", + "num_channels", + "activity_channel", + ): + with self.subTest(name=name): + self.assertFalse(hasattr(params, name)) def test_band_value_params_have_concrete_defaults(self): """Value params default to concrete (non-None) values.""" @@ -70,8 +77,7 @@ def test_band_value_params_have_concrete_defaults(self): self.assertEqual(p.analyze_hz, 100.0) # Seven is the current implementation default, not a new retuning. self.assertEqual(p.peakth, 7.0) - # Only genuinely runtime-resolved fields stay None sentinels. - self.assertIsNone(p.num_channels) + # Only runtime-resolved annotation policy retains a None sentinel. self.assertIsNone(AnnotationOptions().interactive) From 6325551e0f535c46fcf51291f3e05e7c95ddca4b Mon Sep 17 00:00:00 2001 From: Michael Xie Date: Mon, 21 Sep 2026 11:44:46 -0700 Subject: [PATCH 38/55] remove operator parameter --- README.md | 2 +- docs/source/architecture.rst | 2 +- docs/source/migration.rst | 8 ++++++-- examples/extract_band.py | 2 -- src/giant_python/cli.py | 6 ------ src/giant_python/extraction/band/workflow.py | 1 - src/giant_python/gui/parameters_gui.py | 7 +------ src/giant_python/models/params.py | 5 ----- src/giant_python/pipeline/annotate.py | 2 +- tests/test_band_workflow.py | 9 +++++---- tests/test_bandsilo_annotate.py | 2 +- tests/test_cli.py | 17 ++++++++++++++--- tests/test_configuration.py | 19 +++++++++++++------ tests/test_io_edges.py | 6 +++--- tests/test_parameter_gui.py | 16 ++++++++++------ tests/test_staging_example.py | 20 ++++++++++++++++---- tests/test_workflows.py | 2 +- 17 files changed, 73 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index b906b1e..9c71acd 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ The Python API preceding this cleanup was unreleased and unused. Compatibility-o - `extract_band_sources(input, params=None, *, execution=None, annotations=None)` accepts an exact filename or loaded `TrialTable`. Use `Pipeline(..., params=...)` or `Pipeline.from_trial_table(..., params=...).extract()` for a session. - `resolve_band_options(params=None, execution=None, annotations=None)` returns fresh validated `(BandSiloParams, ExecutionOptions, AnnotationOptions)` objects. Each argument accepts **only its corresponding dataclass, a dictionary of fields belonging to that scope, or `None`**. Mixed dictionaries, cross-scope fields and unknown keys are rejected; they are not redistributed or converted. Caller inputs are not mutated. -- `params` holds science only; channel count is inferred from acquisition metadata and source activity extraction uses channel zero. `execution` holds `max_workers`, `verbose`, and `max_trials`; `annotations` holds `enabled`, `interactive`, and `operator`. Generic services and `Pipeline` take `microscope` / `scan_mode` as explicit routing options; only SLAP2 band extraction is implemented (`scan_mode=None` also selects band). There is no registration configuration class. +- `params` holds science only; channel count is inferred from acquisition metadata and source activity extraction uses channel zero. `execution` holds `max_workers`, `verbose`, and `max_trials`; `annotations` holds `enabled` and `interactive`. Generic services and `Pipeline` take `microscope` / `scan_mode` as explicit routing options; only SLAP2 band extraction is implemented (`scan_mode=None` also selects band). There is no registration configuration class. - All per-path results are accessed through `ExperimentSummary.paths`. A `PathSummary` owns `sources`, `user_rois`, `visualizations`, `frame_info`, `global_f`, and `z_depths`. There are no flat or aggregate summary properties. Access frame fields as attributes, such as `path.frame_info.offlineXshifts`, or call the shallow `FrameInfo.to_dict()`; `FrameInfo` is not a mapping. - `giant_python.io.read_summary(path)` returns an `ExperimentSummary`; `giant_python.io.write_summary(summary, path)` accepts that model and returns `None`. Their owner is `giant_python.io.experiment_summary`; the model's `from_h5()` / `to_h5()` delegate there. No public assembled-dictionary writer or alternate summary IO alias is supported. - `giant_python.extraction.band.result_assembly.assemble_path_summary()` is the band result assembly boundary and returns a `PathSummary`. Scientific backend code belongs in `giant_python.extraction.band`; shared kernels belong in `giant_python.numerics`, not the removed compatibility namespaces. diff --git a/docs/source/architecture.rst b/docs/source/architecture.rst index 994d055..f85bfd1 100644 --- a/docs/source/architecture.rst +++ b/docs/source/architecture.rst @@ -77,7 +77,7 @@ preparation derives raw and alignment locations from the model's explicit ``BandSiloParams`` owns scientific settings, ``ExecutionOptions`` owns worker count, verbose reporting and the optional trial limit, and -``AnnotationOptions`` owns ROI inclusion, interactivity and operator metadata. +``AnnotationOptions`` owns ROI inclusion and interactivity. Resolution does not mutate the caller's options to store inferred channel counts. ``ResolvedAcquisition`` carries resolved backend metadata separately. Annotation-disabled and enabled-but-empty are distinct states. diff --git a/docs/source/migration.rst b/docs/source/migration.rst index d20b8f2..6b576a5 100644 --- a/docs/source/migration.rst +++ b/docs/source/migration.rst @@ -49,7 +49,7 @@ drawing; missing required annotations fail explicitly. - ``max_workers``, ``verbose``, ``max_trials`` * - ``annotations`` - ``AnnotationOptions`` - - ``enabled``, ``interactive``, ``operator`` + - ``enabled``, ``interactive`` ``resolve_band_options(params=None, execution=None, annotations=None)`` returns fresh validated objects in the order shown above. Each input accepts **only @@ -62,7 +62,11 @@ counts or GUI policy are later resolved. Defaults remain peak threshold 7, analysis rate 100 Hz, decay 0.15 seconds, baseline window 4 seconds, denoise window 1 second, VIF 1.38 and PSF dilation 17. Execution defaults to six workers, no trial limit and verbosity off. Annotation -defaults to disabled with automatic interactivity and operator ``SLAP2 User``. +defaults to disabled with automatic interactivity. +``operator`` has been removed from ``AnnotationOptions``, the parameter GUI, +the CLI and the staging example. Supplying it as an annotation option raises +``TypeError``; ``--operator`` is no longer accepted. New extraction summaries +do not emit operator metadata; existing summary metadata remains readable. ``num_channels`` and ``activity_channel`` have been removed from ``BandSiloParams``. Channel count is inferred from acquisition metadata and retained in output metadata; source activity extraction always uses channel diff --git a/examples/extract_band.py b/examples/extract_band.py index da2ea5b..81f2333 100644 --- a/examples/extract_band.py +++ b/examples/extract_band.py @@ -101,7 +101,6 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--verbose", action="store_true") parser.add_argument("--use-rois", action="store_true") parser.add_argument("--interactive", action="store_true") - parser.add_argument("--operator", default="SLAP2 User") parser.add_argument("--analyze-hz", type=float, default=100.0) parser.add_argument("--decay-tau-s", type=float, default=0.15) parser.add_argument("--baseline-window-s", type=float, default=4.0) @@ -243,7 +242,6 @@ def prepare_run(args: argparse.Namespace) -> PreparedRun: annotations = AnnotationOptions( enabled=args.use_rois, interactive=args.interactive, - operator=args.operator, ) return PreparedRun(paths, table, params, execution, annotations) diff --git a/src/giant_python/cli.py b/src/giant_python/cli.py index f71d94b..8a97291 100644 --- a/src/giant_python/cli.py +++ b/src/giant_python/cli.py @@ -58,11 +58,6 @@ def _add_silo_options(parser: argparse.ArgumentParser) -> None: default=None, help="Never open a GUI; fail fast if annotations are missing.", ) - parser.add_argument( - "--operator", - default=AnnotationOptions.operator, - help="Operator name recorded in the output metadata.", - ) parser.add_argument( "--verbose", action="store_true", @@ -122,7 +117,6 @@ def _options_from_args( AnnotationOptions( enabled=args.draw_user_rois or args.command == "annotate", interactive=args.interactive, - operator=args.operator, ), ) diff --git a/src/giant_python/extraction/band/workflow.py b/src/giant_python/extraction/band/workflow.py index 036d626..1dd2599 100644 --- a/src/giant_python/extraction/band/workflow.py +++ b/src/giant_python/extraction/band/workflow.py @@ -161,7 +161,6 @@ def _params_dict( "peakth": params.peakth, "peak_buffer": params.peak_buffer, "draw_user_rois": annotations.enabled, - "operator": annotations.operator, } diff --git a/src/giant_python/gui/parameters_gui.py b/src/giant_python/gui/parameters_gui.py index c9b8954..8fe6209 100644 --- a/src/giant_python/gui/parameters_gui.py +++ b/src/giant_python/gui/parameters_gui.py @@ -25,7 +25,6 @@ ("peakth", "Peak Threshold:"), ("peak_buffer", "Peak Buffer:"), ("max_workers", "Max Workers:"), - ("operator", "Operator:"), ) @@ -93,11 +92,7 @@ def on_ok(): result["options"] = ( replace(base, **updates), replace(run, max_workers=int(entries["max_workers"].get())), - replace( - roi, - operator=entries["operator"].get(), - enabled=bool(draw_rois_var.get()), - ), + replace(roi, enabled=bool(draw_rois_var.get())), ) root.destroy() except ValueError as error: diff --git a/src/giant_python/models/params.py b/src/giant_python/models/params.py index cd490ca..ea0d532 100644 --- a/src/giant_python/models/params.py +++ b/src/giant_python/models/params.py @@ -127,13 +127,10 @@ class AnnotationOptions: Include user ROIs during extraction. interactive : bool or None Allow or forbid drawing; ``None`` retains backend auto-detection. - operator : str - Operator name recorded in metadata. """ enabled: bool = False interactive: Optional[bool] = None - operator: str = "SLAP2 User" def __post_init__(self) -> None: """Validate the explicit and automatic annotation policies.""" @@ -143,8 +140,6 @@ def __post_init__(self) -> None: self.interactive, bool ): raise ValueError("interactive must be a bool or None") - if not isinstance(self.operator, str): - raise ValueError("operator must be a string") BandParamsInput = Optional[Union[BandSiloParams, dict]] diff --git a/src/giant_python/pipeline/annotate.py b/src/giant_python/pipeline/annotate.py index 48d88ea..0e2f2a2 100644 --- a/src/giant_python/pipeline/annotate.py +++ b/src/giant_python/pipeline/annotate.py @@ -37,7 +37,7 @@ def annotate_rois( execution : ExecutionOptions, dict or None Run policy, including logging. annotations : AnnotationOptions, dict or None - GUI policy and operator metadata. Calling this explicit annotation + ROI inclusion and GUI policy. Calling this explicit annotation step enables ROI handling for this call even if ``enabled=False``. Returns diff --git a/tests/test_band_workflow.py b/tests/test_band_workflow.py index a43490f..4b8ff62 100644 --- a/tests/test_band_workflow.py +++ b/tests/test_band_workflow.py @@ -45,9 +45,7 @@ def _run_mock_session(self, *, loaded=False, rois=False): frames = np.arange(n_frames, dtype=float) + 11 psf = np.ones((3, 3), dtype=np.float32) options = ExecutionOptions(max_workers=3, max_trials=2) - annotation = AnnotationOptions( - enabled=rois, interactive=False, operator="workflow test" - ) + annotation = AnnotationOptions(enabled=rois, interactive=False) science_args = dict( analyze_hz=10, denoise_window_s=0.3, baseline_window_s=0.5 ) @@ -325,8 +323,11 @@ def read_trial( restored = read_summary(output) self.assertEqual(len(summary.paths), 2) self.assertEqual(len(restored.paths), 2) + assert summary.params is not None + assert restored.params is not None self.assertEqual(summary.params["numChannels"], 2) - self.assertEqual(summary.params["operator"], "workflow test") + self.assertNotIn("operator", summary.params) + self.assertNotIn("operator", restored.params) self.assertEqual(summary.params["draw_user_rois"], rois) for dmd_ix, (path, saved) in enumerate( zip(summary.paths, restored.paths) diff --git a/tests/test_bandsilo_annotate.py b/tests/test_bandsilo_annotate.py index fd4543b..67828d7 100644 --- a/tests/test_bandsilo_annotate.py +++ b/tests/test_bandsilo_annotate.py @@ -164,7 +164,7 @@ def test_split_options_and_in_memory_input(self): input = TrialTable() params = BandSiloParams() execution = ExecutionOptions(verbose=True) - annotations = AnnotationOptions(interactive=False, operator="Ada") + annotations = AnnotationOptions(interactive=False) before = deepcopy((params, execution, annotations)) table = { "fn_adata": np.array([["a.h5"]]), diff --git a/tests/test_cli.py b/tests/test_cli.py index a11a788..7037104 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -39,6 +39,19 @@ def test_removed_commands_are_absent(self): self.assertIn("invalid choice", stderr.getvalue()) self.assertIn(command, stderr.getvalue()) + def test_operator_flag_is_rejected(self): + """Neither command accepts the removed operator option.""" + for command in ("annotate", "extract"): + with self.subTest(command=command): + with mock.patch("sys.stderr", new_callable=io.StringIO): + with self.assertRaises(SystemExit) as error: + self.parser.parse_args( + [command, "tt.h5", "--operator", "Ada"] + ) + self.assertEqual(error.exception.code, 2) + args = self.parser.parse_args([command, "tt.h5"]) + self.assertFalse(hasattr(args, "operator")) + def test_annotate_defaults(self): """annotate defaults to band scan, no draw, auto interactivity.""" args = self.parser.parse_args(["annotate", "tt.h5"]) @@ -94,8 +107,6 @@ def test_maps_fields(self): "band", "--draw-user-rois", "--interactive", - "--operator", - "Alice", "--verbose", ] ) @@ -104,7 +115,7 @@ def test_maps_fields(self): self.assertFalse(hasattr(params, "scan_mode")) self.assertTrue(annotations.enabled) self.assertTrue(annotations.interactive) - self.assertEqual(annotations.operator, "Alice") + self.assertFalse(hasattr(annotations, "operator")) self.assertTrue(execution.verbose) self.assertEqual(execution.max_workers, 6) self.assertIsNone(execution.max_trials) diff --git a/tests/test_configuration.py b/tests/test_configuration.py index ef9ac08..c14d7f4 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -23,9 +23,8 @@ def test_defaults_match_existing_science(self): self.assertEqual(science.vif, 1.38) self.assertEqual(science.sparse_fac, math.exp(-3.0)) self.assertEqual(execution, ExecutionOptions(6, False, None)) - self.assertEqual( - annotations, AnnotationOptions(False, None, "SLAP2 User") - ) + self.assertEqual(annotations, AnnotationOptions(False, None)) + self.assertFalse(hasattr(annotations, "operator")) self.assertFalse(hasattr(science, "max_workers")) self.assertFalse(hasattr(science, "scan_mode")) self.assertFalse(hasattr(science, "interactive")) @@ -68,7 +67,6 @@ def test_dictionary_inputs_and_overrides_are_not_modified(self): annotations = { "enabled": True, "interactive": False, - "operator": "Grace", } original = deepcopy((params, execution, annotations)) science, run, roi = resolve_band_options( @@ -76,7 +74,7 @@ def test_dictionary_inputs_and_overrides_are_not_modified(self): ) self.assertEqual(science.analyze_hz, 80.0) self.assertEqual(run, ExecutionOptions(2, True, None)) - self.assertEqual(roi, AnnotationOptions(True, False, "Grace")) + self.assertEqual(roi, AnnotationOptions(True, False)) self.assertEqual((params, execution, annotations), original) @@ -136,6 +134,16 @@ def test_channel_parameters_are_not_configurable(self): with self.assertRaisesRegex(TypeError, name): resolve_band_options(options) + def test_operator_is_not_configurable(self): + """Removed operator metadata is rejected rather than ignored.""" + for value in ("Ada", "SLAP2 User", None): + with self.subTest(value=value): + options = {"operator": value} + with self.assertRaisesRegex(TypeError, "operator"): + AnnotationOptions(**options) + with self.assertRaisesRegex(TypeError, "operator"): + resolve_band_options(annotations=options) + def test_execution_and_annotation_policy(self): """Execution counts and tri-state GUI settings are validated.""" for name in ("max_workers", "max_trials"): @@ -148,7 +156,6 @@ def test_execution_and_annotation_policy(self): for name, value in ( ("enabled", 1), ("interactive", "false"), - ("operator", None), ): with self.subTest(name=name): with self.assertRaisesRegex(ValueError, name): diff --git a/tests/test_io_edges.py b/tests/test_io_edges.py index 4e3ef28..9984c0e 100644 --- a/tests/test_io_edges.py +++ b/tests/test_io_edges.py @@ -515,9 +515,9 @@ def test_geometry_uses_first_kept_trial_for_each_dmd(self): def test_annotation_option_resolution_preserves_input(self): """Annotation uses scoped option conversion without mutation.""" - original = AnnotationOptions(operator="operator μ", interactive=False) + original = AnnotationOptions(interactive=False) _, _, resolved = annotation.resolve_band_options(annotations=original) - self.assertEqual(resolved.operator, "operator μ") + self.assertEqual(resolved, original) + self.assertIsNot(resolved, original) self.assertFalse(resolved.interactive) - self.assertEqual(original.operator, "operator μ") self.assertFalse(original.interactive) diff --git a/tests/test_parameter_gui.py b/tests/test_parameter_gui.py index dc93654..b31ae8c 100644 --- a/tests/test_parameter_gui.py +++ b/tests/test_parameter_gui.py @@ -1,5 +1,6 @@ """Headless checks for the parameter form's canonical split-option API.""" +import math import sys import unittest from unittest.mock import MagicMock, patch @@ -50,13 +51,13 @@ def test_success_returns_fresh_split_options(self): """Scientific, execution, and annotation controls stay independent.""" params = BandSiloParams(peakth=8) run = ExecutionOptions(max_workers=2, verbose=True, max_trials=4) - roi = AnnotationOptions(interactive=False, operator="original") + roi = AnnotationOptions(interactive=False) def interact(): """Simulate edits to a scientific field and both policy groups.""" self.variables[0].get.return_value = "125" self.variables[8].get.return_value = "3" - self.variables[9].get.return_value = "reviewer" + self.variables[9].get.return_value = "-2" self.roi_var.get.return_value = True self.buttons["OK"]() @@ -66,14 +67,17 @@ def interact(): science, execution, annotations = result self.assertEqual(science.analyze_hz, 125) self.assertEqual(science.peakth, 8) + self.assertAlmostEqual(science.sparse_fac, math.exp(-2)) + labels = [ + call.kwargs["text"] for call in self.tk.ttk.Label.call_args_list + ] + self.assertNotIn("Operator:", labels) self.assertEqual(execution, ExecutionOptions(3, True, 4)) - self.assertEqual( - annotations, AnnotationOptions(True, False, "reviewer") - ) + self.assertEqual(annotations, AnnotationOptions(True, False)) self.assertEqual(params.analyze_hz, 100) self.assertEqual(run.max_workers, 2) self.assertFalse(roi.enabled) - self.assertEqual(roi.operator, "original") + self.assertFalse(roi.interactive) for supplied, returned in zip((params, run, roi), result): self.assertIsNot(supplied, returned) self.root.destroy.assert_called_once() diff --git a/tests/test_staging_example.py b/tests/test_staging_example.py index a95888c..50c8465 100644 --- a/tests/test_staging_example.py +++ b/tests/test_staging_example.py @@ -89,6 +89,7 @@ def test_help_exits_before_validation(self): example.main(["--help"]) self.assertEqual(raised.exception.code, 0) self.assertIn("--stage-data", output.getvalue()) + self.assertNotIn("--operator", output.getvalue()) prepare.assert_not_called() @@ -248,7 +249,7 @@ def test_default_run_relocates_loaded_model_without_rewriting_source(self): self.assertEqual(kwargs["execution"], ExecutionOptions()) self.assertEqual( kwargs["annotations"], - AnnotationOptions(False, False, "SLAP2 User"), + AnnotationOptions(False, False), ) self.assertEqual( table.source_path.read_bytes(), self.table_path.read_bytes() @@ -259,6 +260,19 @@ def test_default_run_relocates_loaded_model_without_rewriting_source(self): ) self.assertFalse((self.results / "input").exists()) + def test_operator_option_is_rejected_before_staging(self): + """The removed CLI option fails before any extraction or staging.""" + before = self.snapshot() + with ( + contextlib.redirect_stderr(io.StringIO()) as stderr, + self.assertRaises(SystemExit) as raised, + ): + self.invoke("--operator", "Ada") + self.assertEqual(raised.exception.code, 2) + self.assertIn("unrecognized arguments: --operator", stderr.getvalue()) + self.assertEqual(self.snapshot(), before) + self.extract.assert_not_called() + def test_copy_stages_sidecars_references_annotations_and_options(self): """Copy input trees and forward split policy/science overrides.""" self.invoke( @@ -272,8 +286,6 @@ def test_copy_stages_sidecars_references_annotations_and_options(self): "--max-trials", 3, "--verbose", - "--operator", - "Ada", "--denoise-window-s", 2, "--vif", @@ -295,7 +307,7 @@ def test_copy_stages_sidecars_references_annotations_and_options(self): self.extract.call_args.kwargs, { "execution": ExecutionOptions(2, True, 3), - "annotations": AnnotationOptions(True, False, "Ada"), + "annotations": AnnotationOptions(True, False), }, ) diff --git a/tests/test_workflows.py b/tests/test_workflows.py index 51a3567..1f3a5b5 100644 --- a/tests/test_workflows.py +++ b/tests/test_workflows.py @@ -174,7 +174,7 @@ def test_annotation_forwards_options_without_mutation(self): """Annotation forwards split options directly to the backend.""" table = TrialTable() params = BandSiloParams() - options = AnnotationOptions(interactive=False, operator="Ada") + options = AnnotationOptions(interactive=False) before = deepcopy((params, options)) result = annotate_rois(table, params, annotations=options) self.assertEqual(result, "annotations.h5") From c5d29bbbd24ef07a6adfa2c99f4f31ad4d6645dd Mon Sep 17 00:00:00 2001 From: m-xie Date: Tue, 29 Sep 2026 23:47:46 +0000 Subject: [PATCH 39/55] allow activity image peak threshold to vary between ROIs --- src/giant_python/numerics/peaks.py | 82 ++++++++++++-------- tests/test_bandsilo_peaks.py | 115 +++++++++++++++++++++++++++++ 2 files changed, 165 insertions(+), 32 deletions(-) diff --git a/src/giant_python/numerics/peaks.py b/src/giant_python/numerics/peaks.py index cd27660..65aa78a 100644 --- a/src/giant_python/numerics/peaks.py +++ b/src/giant_python/numerics/peaks.py @@ -36,6 +36,9 @@ # MAD -> Gaussian-sigma scale factor (median absolute deviation / 0.6745). _MAD_TO_SIGMA = 0.67449 +# Minimum finite-pixel count for an independent background estimate. +_MIN_COMPONENT_PIXELS = 100 + def _validate_theta(theta: np.ndarray) -> None: """Require explicit amplitude, center and both axis-width columns.""" @@ -654,7 +657,7 @@ def detect_peaks_2d( exclusion_mask : ndarray of bool, shape (H, W) Pixels to exclude from detection. mu_bg, sigma_bg : float - Global background median and MAD-scaled sigma. + Background median and MAD-scaled sigma for the supplied region. peak_thresh : float Absolute detection threshold ``mu_bg + peak_th * sigma_bg``. peak_th : float @@ -758,9 +761,12 @@ def get_act_im_peaks( ) -> np.ndarray: """Find Gaussian source seeds in a 3-D (Z, H, W) activity image. - Background statistics (median and MAD-scaled sigma) and the detection - threshold are computed once across all planes for uniform sensitivity, then - each plane is detected independently (:func:`detect_peaks_2d`). + Each plane is split into 4-connected components of finite pixels. Regions + with fewer than 100 pixels or zero MAD are skipped. Background statistics + (median and MAD-scaled sigma) and the detection threshold are computed + independently for each remaining component. Detection and fitting are + restricted to that component (:func:`detect_peaks_2d`). Exclusion masks + suppress peak detection but do not affect connectivity or background stats. Gaussians have independent, axis-aligned standard deviations, bounded to ``0.35 <= sigma_y <= 8`` and ``0.35 <= sigma_x <= 1`` pixels. @@ -771,9 +777,9 @@ def get_act_im_peaks( Parameters ---------- act_im : ndarray of shape (Z, H, W) - Activity image (may contain NaNs). + Activity image; nonfinite pixels separate valid-pixel components. peak_th : float - Threshold in MAD-normalized standard deviations. + Threshold in component-local MAD-normalized standard deviations. exclusion_mask : None or ndarray ``None``, a 2-D ``(H, W)`` mask applied to every plane, or a 3-D ``(Z, H, W)`` per-plane mask (e.g. user soma ROIs). @@ -796,35 +802,47 @@ def get_act_im_peaks( else: excl_planes = [exclusion_mask[z].astype(bool) for z in range(n_z)] - valid_vals = act_im[~np.isnan(act_im)] - if valid_vals.size == 0: - return empty - - mu_bg = float(np.nanmedian(act_im)) - sigma_bg = ( - float(np.median(np.abs(valid_vals - np.median(valid_vals)))) - / _MAD_TO_SIGMA - ) - if sigma_bg <= 0: - return empty - - peak_thresh = mu_bg + peak_th * sigma_bg - source_seeds_list = [] for z in range(n_z): - thetaf_z = detect_peaks_2d( - act_im[z], - excl_planes[z], - mu_bg, - sigma_bg, - peak_thresh, - peak_th, - buffer_size=buffer_size, + labels, _ = cast( + tuple[np.ndarray, int], ndimage.label(np.isfinite(act_im[z])) ) - if thetaf_z.shape[0] > 0: - z_col = np.full((thetaf_z.shape[0], 1), z, dtype=float) - source_seeds_list.append( - np.column_stack([z_col, thetaf_z[:, 1], thetaf_z[:, 2]]) + counts = np.bincount(labels.ravel()) + for label, bounds in enumerate(ndimage.find_objects(labels), start=1): + if bounds is None or counts[label] < _MIN_COMPONENT_PIXELS: + continue + + component = labels[bounds] == label + plane_crop = act_im[z][bounds] + valid_vals = plane_crop[component] + mu_bg = float(np.median(valid_vals)) + sigma_bg = ( + float(np.median(np.abs(valid_vals - mu_bg))) / _MAD_TO_SIGMA + ) + if sigma_bg <= 0: + continue + + # NaN-mask other regions, including any inside this bounding box, + # so neither the initial fit nor residual refinement uses them. + thetaf = detect_peaks_2d( + np.where(component, plane_crop, np.nan), + excl_planes[z][bounds], + mu_bg, + sigma_bg, + mu_bg + peak_th * sigma_bg, + peak_th, + buffer_size=buffer_size, ) + if thetaf.shape[0] > 0: + z_col = np.full(thetaf.shape[0], z, dtype=float) + source_seeds_list.append( + np.column_stack( + [ + z_col, + thetaf[:, 1] + bounds[0].start, + thetaf[:, 2] + bounds[1].start, + ] + ) + ) return np.vstack(source_seeds_list) if source_seeds_list else empty diff --git a/tests/test_bandsilo_peaks.py b/tests/test_bandsilo_peaks.py index 8a40524..b45f8d7 100644 --- a/tests/test_bandsilo_peaks.py +++ b/tests/test_bandsilo_peaks.py @@ -539,6 +539,121 @@ def test_close_pair_adds_residual_peak(self): class TestGetActImPeaks(unittest.TestCase): """get_act_im_peaks 3-D driver and its guard branches.""" + def test_component_local_statistics_and_coordinates(self): + """Each component and plane gets its own median, MAD and threshold.""" + act = np.full((2, 16, 29), np.nan) + base = np.linspace(-1.0, 1.0, 100).reshape(10, 10) + regions = [ + (0, 2, 3, base), + (0, 4, 18, 5.0 + 8.0 * base), + (1, 2, 3, 2.0 + 3.0 * base), + ] + for z, y, x, values in regions: + act[z, y : y + 10, x : x + 10] = values + original = act.copy() + fitted = np.array([[10.0, 4.2, 5.3, 0.5, 0.5]]) + with patch.object( + pk, "detect_peaks_2d", return_value=fitted + ) as detect: + seeds = pk.get_act_im_peaks(act, peak_th=4.0, buffer_size=3) + + self.assertEqual(detect.call_count, 3) + expected_seeds = [] + for call, (z, y, x, values) in zip(detect.call_args_list, regions): + image, mask, mu, sigma, threshold, peak_th = call.args + expected_mu = np.median(values) + expected_sigma = np.median(np.abs(values - expected_mu)) / 0.67449 + np.testing.assert_array_equal(image, values) + self.assertFalse(mask.any()) + self.assertAlmostEqual(mu, expected_mu) + self.assertAlmostEqual(sigma, expected_sigma) + self.assertAlmostEqual(threshold, expected_mu + 4 * expected_sigma) + self.assertEqual(peak_th, 4.0) + self.assertEqual(call.kwargs, {"buffer_size": 3}) + expected_seeds.append([z, y + 4.2, x + 5.3]) + np.testing.assert_allclose(seeds, expected_seeds) + np.testing.assert_array_equal(act, original) + + def test_minimum_size_is_per_plane_and_inclusive(self): + """99-pixel regions are skipped even across z; 100 pixels qualifies.""" + act = np.full((2, 12, 23), np.nan) + act[:, :9, :11] = np.arange(99).reshape(9, 11) + act[0, :10, 13:23] = np.arange(100).reshape(10, 10) + with patch.object( + pk, "detect_peaks_2d", return_value=np.zeros((0, 5)) + ) as detect: + seeds = pk.get_act_im_peaks(act) + self.assertEqual(detect.call_count, 1) + self.assertEqual(detect.call_args.args[0].shape, (10, 10)) + self.assertEqual(seeds.shape, (0, 3)) + + def test_diagonal_regions_are_not_connected(self): + """Diagonal contact does not merge two sub-100-pixel regions.""" + act = np.full((1, 18, 18), np.nan) + act[0, :9, :9] = np.arange(81).reshape(9, 9) + act[0, 9:, 9:] = np.arange(81).reshape(9, 9) + with patch.object(pk, "detect_peaks_2d") as detect: + seeds = pk.get_act_im_peaks(act) + detect.assert_not_called() + self.assertEqual(seeds.shape, (0, 3)) + + def test_nested_components_and_exclusion_masks(self): + """Regions never share fit pixels; exclusions do not split CCs.""" + act = np.arange(900, dtype=float).reshape(1, 30, 30) + act[0, 8:22, 8:22] = np.inf + act[0, 10:20, 10:20] = np.arange(100).reshape(10, 10) + outer = np.isfinite(act[0]) + outer[10:20, 10:20] = False + for shape in [(30, 30), (1, 30, 30)]: + with self.subTest(mask_shape=shape): + exclusion = np.zeros(shape, dtype=bool) + exclusion[..., 14:16, :] = True + mask = exclusion if exclusion.ndim == 2 else exclusion[0] + with patch.object( + pk, "detect_peaks_2d", return_value=np.zeros((0, 5)) + ) as detect: + pk.get_act_im_peaks(act, exclusion_mask=exclusion) + self.assertEqual(detect.call_count, 2) + outer_call, inner_call = detect.call_args_list + np.testing.assert_array_equal( + np.isfinite(outer_call.args[0]), outer + ) + np.testing.assert_array_equal(outer_call.args[1], mask) + self.assertAlmostEqual( + outer_call.args[2], np.median(act[0][outer]) + ) + np.testing.assert_array_equal( + inner_call.args[0], act[0, 10:20, 10:20] + ) + np.testing.assert_array_equal( + inner_call.args[1], mask[10:20, 10:20] + ) + + def test_zero_mad_component_does_not_skip_other_components(self): + """A constant region is skipped independently of a noisy region.""" + act = np.ones((1, 10, 21)) + act[0, :, 10] = np.nan + act[0, :, 11:] = np.arange(100).reshape(10, 10) + with patch.object( + pk, "detect_peaks_2d", return_value=np.zeros((0, 5)) + ) as detect: + pk.get_act_im_peaks(act) + self.assertEqual(detect.call_count, 1) + self.assertAlmostEqual(detect.call_args.args[2], 49.5) + + def test_quiet_component_peak_survives_noisy_neighbor(self): + """A large noisy region cannot hide a quiet region's genuine peak.""" + rng = np.random.default_rng(17) + act = np.full((1, 80, 112), np.nan) + act[0, 5:35, 4:34] = rng.normal(0, 0.03, (30, 30)) + act[0, 5:35, 4:34] += _single_gaussian_plane( + 30, 30, 14.3, 16.7, 3.0, 0.8 + ) + act[0, :, 36:] = rng.normal(0, 2.0, (80, 76)) + seeds = pk.get_act_im_peaks(act, peak_th=6.0, buffer_size=3) + self.assertEqual(seeds.shape, (1, 3)) + np.testing.assert_allclose(seeds[0], [0, 19.3, 20.7], atol=0.15) + def test_elongated_blob_seed(self): """The 3-D driver retains its [z, y, x] output for unequal widths.""" rng = np.random.default_rng(42) From fb148c7c572b84c22be602d60adc5f42935a8bd6 Mon Sep 17 00:00:00 2001 From: m-xie Date: Wed, 30 Sep 2026 00:55:18 +0000 Subject: [PATCH 40/55] add z-tolerance parameter --- README.md | 6 ++++++ examples/extract_band.py | 3 +++ src/giant_python/cli.py | 8 +++++++- src/giant_python/extraction/band/workflow.py | 2 +- src/giant_python/gui/parameters_gui.py | 2 ++ src/giant_python/models/params.py | 5 +++++ tests/test_band_workflow.py | 12 +++++++++++- tests/test_bandsilo_background.py | 13 +++++++++++++ tests/test_cli.py | 4 ++++ tests/test_configuration.py | 5 ++++- tests/test_parameter_gui.py | 5 ++++- 11 files changed, 60 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 9c71acd..bb0a7bb 100644 --- a/README.md +++ b/README.md @@ -100,6 +100,12 @@ The `giant` CLI exposes only `annotate` and `extract`, each taking an exact tria ### Canonical API and ownership +`BandSiloParams(z_tol=1.5)` controls the z tolerance in microns for low-resolution +motion-bin selection: bins must lie within this absolute distance of the median +frame z and still satisfy the existing minimum frame count. The value must be +positive and finite; the default preserves previous behavior. The CLI and staging +example accept `--z-tol`, and the parameter GUI labels it **Z Tolerance (µm)**. + The Python API preceding this cleanup was unreleased and unused. Compatibility-only Python imports and adapters have been removed, not deprecated. MATLAB file-format compatibility is retained separately; there is no production Python API migration requirement. - `extract_band_sources(input, params=None, *, execution=None, annotations=None)` accepts an exact filename or loaded `TrialTable`. Use `Pipeline(..., params=...)` or `Pipeline.from_trial_table(..., params=...).extract()` for a session. diff --git a/examples/extract_band.py b/examples/extract_band.py index 81f2333..a18eb75 100644 --- a/examples/extract_band.py +++ b/examples/extract_band.py @@ -109,6 +109,8 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--d-xy", type=int, default=5) parser.add_argument("--peak-buffer", type=int, default=3) parser.add_argument("--psf-dilation", type=int, default=17) + parser.add_argument("--z-tol", type=float, default=1.5, + help="Z tolerance for motion-bin selection (microns)") return parser @@ -233,6 +235,7 @@ def prepare_run(args: argparse.Namespace) -> PreparedRun: d_xy=args.d_xy, peak_buffer=args.peak_buffer, psf_dilation=args.psf_dilation, + z_tol=args.z_tol, ) execution = ExecutionOptions( max_workers=args.max_workers, diff --git a/src/giant_python/cli.py b/src/giant_python/cli.py index 8a97291..f7e4e3f 100644 --- a/src/giant_python/cli.py +++ b/src/giant_python/cli.py @@ -26,6 +26,12 @@ def _add_silo_options(parser: argparse.ArgumentParser) -> None: """Attach the shared source-extraction options to a subparser.""" + parser.add_argument( + "--z-tol", + type=float, + default=BandSiloParams.z_tol, + help="Z tolerance for motion-bin selection in microns (default: 1.5).", + ) parser.add_argument( "--microscope", choices=("slap2",), @@ -108,7 +114,7 @@ def _options_from_args( ) -> tuple[BandSiloParams, ExecutionOptions, AnnotationOptions]: """Build independent science, execution and annotation configuration.""" return ( - BandSiloParams(), + BandSiloParams(z_tol=args.z_tol), ExecutionOptions( max_workers=args.max_workers, verbose=args.verbose, diff --git a/src/giant_python/extraction/band/workflow.py b/src/giant_python/extraction/band/workflow.py index 1dd2599..975aa60 100644 --- a/src/giant_python/extraction/band/workflow.py +++ b/src/giant_python/extraction/band/workflow.py @@ -246,7 +246,7 @@ def _read(trial_ix): ) median_z = np.median(low["lowResMotionZ"]) _, frames_to_keep = motion_binning.select_motion_bins( - unique_motion, mot_inds, low["lowResMotionZ"] + unique_motion, mot_inds, low["lowResMotionZ"], z_thresh=params.z_tol ) mean_im = si.compute_mean_image( low_res_data_norm, diff --git a/src/giant_python/gui/parameters_gui.py b/src/giant_python/gui/parameters_gui.py index 8fe6209..79faefc 100644 --- a/src/giant_python/gui/parameters_gui.py +++ b/src/giant_python/gui/parameters_gui.py @@ -25,6 +25,7 @@ ("peakth", "Peak Threshold:"), ("peak_buffer", "Peak Buffer:"), ("max_workers", "Max Workers:"), + ("z_tol", "Z Tolerance (µm):"), ) @@ -87,6 +88,7 @@ def on_ok(): "d_xy": int(entries["d_xy"].get()), "peakth": float(entries["peakth"].get()), "peak_buffer": int(entries["peak_buffer"].get()), + "z_tol": float(entries["z_tol"].get()), "sparse_fac": float(np.exp(float(sparse_var.get()))), } result["options"] = ( diff --git a/src/giant_python/models/params.py b/src/giant_python/models/params.py index ea0d532..7c05cba 100644 --- a/src/giant_python/models/params.py +++ b/src/giant_python/models/params.py @@ -53,6 +53,9 @@ class BandSiloParams: psf_dilation : int Positive dilation size selecting the bundled PSF template. Template availability is checked by the backend when loading the PSF. + z_tol : float + Positive z tolerance in microns: maximum absolute motion-bin z + deviation from the median frame z for low-resolution frame selection. Notes ----- @@ -73,6 +76,7 @@ class BandSiloParams: peakth: float = 7.0 peak_buffer: int = 3 psf_dilation: int = 17 + z_tol: float = 1.5 def __post_init__(self) -> None: """Validate values supported by the existing numerical path.""" @@ -84,6 +88,7 @@ def __post_init__(self) -> None: "vif", "sparse_fac", "peakth", + "z_tol", ): _positive(name, getattr(self, name)) for name in ("d_xy", "peak_buffer", "psf_dilation"): diff --git a/tests/test_band_workflow.py b/tests/test_band_workflow.py index 4b8ff62..04162da 100644 --- a/tests/test_band_workflow.py +++ b/tests/test_band_workflow.py @@ -47,7 +47,8 @@ def _run_mock_session(self, *, loaded=False, rois=False): options = ExecutionOptions(max_workers=3, max_trials=2) annotation = AnnotationOptions(enabled=rois, interactive=False) science_args = dict( - analyze_hz=10, denoise_window_s=0.3, baseline_window_s=0.5 + analyze_hz=10, denoise_window_s=0.3, baseline_window_s=0.5, + z_tol=2.5, ) params = BandSiloParams(**science_args) before = (asdict(params), asdict(options), asdict(annotation)) @@ -271,6 +272,12 @@ def read_trial( wf, "assemble_path_summary", wraps=wf.assemble_path_summary ) ) + select_bins = stack.enter_context( + patch.object( + wf.motion_binning, "select_motion_bins", + wraps=wf.motion_binning.select_motion_bins, + ) + ) writer = stack.enter_context( patch.object( wf, @@ -302,6 +309,9 @@ def read_trial( str(table.savedr / "motion_correction" / "p0-selected.h5"), ) self.assertEqual(low_read.call_count, 4) + self.assertEqual(select_bins.call_count, 2) + for call in select_bins.call_args_list: + self.assertEqual(call.kwargs["z_thresh"], params.z_tol) self.assertEqual(high_read.call_count, 2) self.assertEqual(mapper.call_count, 4) for call in mapper.call_args_list: diff --git a/tests/test_bandsilo_background.py b/tests/test_bandsilo_background.py index 3bfd0c4..5e675e7 100644 --- a/tests/test_bandsilo_background.py +++ b/tests/test_bandsilo_background.py @@ -131,6 +131,19 @@ def test_select_motion_bins_filters_z_and_count(self): self.assertEqual(list(keep), [0]) self.assertEqual(int(frames.sum()), 5) + def test_z_tolerance_changes_selected_bins(self): + """Tolerance includes its boundary without relaxing frame counts.""" + unique_motion = np.array([[0, 0, 0], [0, 0, 2], [1, 0, 0]]) + mot_inds = np.array([0] * 202 + [1] * 101 + [2] * 100) + motion_z = unique_motion[mot_inds, 2] + for tolerance, expected in ((1.5, [0]), (2.0, [0, 1])): + with self.subTest(tolerance=tolerance): + keep, frames = motion_binning.select_motion_bins( + unique_motion, mot_inds, motion_z, z_thresh=tolerance + ) + np.testing.assert_array_equal(keep, expected) + np.testing.assert_array_equal(frames, np.isin(mot_inds, expected)) + def test_bin_motion_yx_marks_dropped_frames(self): """Frames outside frames_to_keep get index -1.""" mr = np.array([0.0, 0.0, 1.0, 1.0]) diff --git a/tests/test_cli.py b/tests/test_cli.py index 7037104..650b533 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -112,6 +112,7 @@ def test_maps_fields(self): ) params, execution, annotations = cli._options_from_args(args) self.assertEqual(params.peakth, 7.0) + self.assertEqual(params.z_tol, 1.5) self.assertFalse(hasattr(params, "scan_mode")) self.assertTrue(annotations.enabled) self.assertTrue(annotations.interactive) @@ -204,12 +205,15 @@ def test_extract_service(self): "curated_v2.h5", "--max-trials", "4", + "--z-tol", + "2.5", "--headless", "--draw-user-rois", ] ) self.assertEqual(code, 0) self.assertEqual(extract.call_args.args[0], "curated_v2.h5") + self.assertEqual(extract.call_args.args[1].z_tol, 2.5) self.assertEqual(extract.call_args.kwargs["scan_mode"], "band") self.assertEqual(extract.call_args.kwargs["execution"].max_trials, 4) self.assertTrue(extract.call_args.kwargs["annotations"].enabled) diff --git a/tests/test_configuration.py b/tests/test_configuration.py index c14d7f4..503b5d2 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -21,6 +21,7 @@ def test_defaults_match_existing_science(self): self.assertEqual(science.peakth, 7.0) self.assertEqual(science.denoise_window_s, 1.0) self.assertEqual(science.vif, 1.38) + self.assertEqual(science.z_tol, 1.5) self.assertEqual(science.sparse_fac, math.exp(-3.0)) self.assertEqual(execution, ExecutionOptions(6, False, None)) self.assertEqual(annotations, AnnotationOptions(False, None)) @@ -62,7 +63,7 @@ def test_every_result_is_fresh(self): def test_dictionary_inputs_and_overrides_are_not_modified(self): """Each dictionary affects only its scope and is copied.""" - params = {"analyze_hz": 80.0} + params = {"analyze_hz": 80.0, "z_tol": 2.5} execution = {"max_workers": 2, "verbose": True} annotations = { "enabled": True, @@ -73,6 +74,7 @@ def test_dictionary_inputs_and_overrides_are_not_modified(self): params, execution, annotations ) self.assertEqual(science.analyze_hz, 80.0) + self.assertEqual(science.z_tol, 2.5) self.assertEqual(run, ExecutionOptions(2, True, None)) self.assertEqual(roi, AnnotationOptions(True, False)) self.assertEqual((params, execution, annotations), original) @@ -91,6 +93,7 @@ def test_positive_scientific_values(self): "vif", "sparse_fac", "peakth", + "z_tol", ): for value in (0, -1, math.nan, math.inf, True, "1", None): with self.subTest(name=name, value=value): diff --git a/tests/test_parameter_gui.py b/tests/test_parameter_gui.py index b31ae8c..4595880 100644 --- a/tests/test_parameter_gui.py +++ b/tests/test_parameter_gui.py @@ -57,7 +57,8 @@ def interact(): """Simulate edits to a scientific field and both policy groups.""" self.variables[0].get.return_value = "125" self.variables[8].get.return_value = "3" - self.variables[9].get.return_value = "-2" + self.variables[9].get.return_value = "2.5" + self.variables[10].get.return_value = "-2" self.roi_var.get.return_value = True self.buttons["OK"]() @@ -67,6 +68,8 @@ def interact(): science, execution, annotations = result self.assertEqual(science.analyze_hz, 125) self.assertEqual(science.peakth, 8) + self.assertEqual(science.z_tol, 2.5) + self.assertEqual(params.z_tol, 1.5) self.assertAlmostEqual(science.sparse_fac, math.exp(-2)) labels = [ call.kwargs["text"] for call in self.tk.ttk.Label.call_args_list From 7ca8fe9641fca2e3b9ab75f8c21667f061c3fe92 Mon Sep 17 00:00:00 2001 From: m-xie Date: Wed, 30 Sep 2026 08:11:52 +0000 Subject: [PATCH 41/55] update z-tolerance in all steps --- .../extraction/band/motion_binning.py | 8 +++---- src/giant_python/extraction/band/traces.py | 15 ++++++++++-- src/giant_python/extraction/band/workflow.py | 3 ++- tests/test_band_workflow.py | 11 ++++++++- tests/test_bandsilo_background.py | 4 ++-- tests/test_bandsilo_traces.py | 23 +++++++++++++++++++ 6 files changed, 54 insertions(+), 10 deletions(-) diff --git a/src/giant_python/extraction/band/motion_binning.py b/src/giant_python/extraction/band/motion_binning.py index 43d4055..91ff94d 100644 --- a/src/giant_python/extraction/band/motion_binning.py +++ b/src/giant_python/extraction/band/motion_binning.py @@ -38,12 +38,12 @@ def select_motion_bins( unique_motion: np.ndarray, mot_inds: np.ndarray, motion_z: np.ndarray, - z_thresh: float = 1.5, + z_tol: float = 1.5, min_frames: int = 100, ) -> Tuple[np.ndarray, np.ndarray]: """Keep motion bins near the median z with enough frames. - A bin is kept when its z is within ``z_thresh`` of the median z (over all + A bin is kept when its z is within ``z_tol`` of the median z (over all frames) and it contains more than ``min_frames`` frames. After filtering, the remaining frames are treated as having no z motion. @@ -55,7 +55,7 @@ def select_motion_bins( Per-frame motion-bin indices. motion_z : ndarray Per-frame z motion (used for the median). - z_thresh : float + z_tol : float Maximum absolute z deviation from the median, in microns. min_frames : int Minimum frame count (strictly greater) for a bin to be kept. @@ -69,7 +69,7 @@ def select_motion_bins( """ median_z = np.median(motion_z) bin_counts = np.bincount(mot_inds, minlength=unique_motion.shape[0]) - keep_mask = (np.abs(unique_motion[:, 2] - median_z) <= z_thresh) & ( + keep_mask = (np.abs(unique_motion[:, 2] - median_z) <= z_tol) & ( bin_counts > min_frames ) mot_inds_to_keep = np.nonzero(keep_mask)[0] diff --git a/src/giant_python/extraction/band/traces.py b/src/giant_python/extraction/band/traces.py index e18bcd1..f351b20 100644 --- a/src/giant_python/extraction/band/traces.py +++ b/src/giant_python/extraction/band/traces.py @@ -123,10 +123,11 @@ def _bin_trial_motion( median_z: float, unique_motion_ds: np.ndarray, mot_inds_to_keep_ds: np.ndarray, + z_tol: float = 1.5, ) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: """Bin trial frames by motion and match them to the low-res motion bins. - Frames within 1.5 um of the median z are rebinned into 2-D (row, col) + Frames within ``z_tol`` um of the median z are rebinned into 2-D (row, col) motion bins; each retained low-res bin is matched to its trial bin so the same PSF geometry is reused. @@ -140,6 +141,8 @@ def _bin_trial_motion( The low-res kept 2-D motion vectors. mot_inds_to_keep_ds : ndarray Indices of the low-res bins to keep. + z_tol : float, optional + Maximum absolute rounded z offset from the median, in um (default 1.5). Returns ------- @@ -160,7 +163,7 @@ def _bin_trial_motion( mot_inds = np.reshape(mot_inds, -1) frames_to_keep = np.isin( mot_inds, - np.flatnonzero(np.abs(unique_motion3[:, 2] - median_z) <= 1.5), + np.flatnonzero(np.abs(unique_motion3[:, 2] - median_z) <= z_tol), ) mot_inds = -1 * np.ones((len(mot_inds),), dtype=np.int32) @@ -362,6 +365,7 @@ def compute_high_res_traces( dmd_pixels_per_row: int, num_channels: int, soma_sps: List[np.ndarray], + z_tol: float = 1.5, ) -> TrialTraceResult: """Compute one trial's high-res source traces from loaded arrays. @@ -398,6 +402,8 @@ def compute_high_res_traces( Number of acquisition channels. soma_sps : list of ndarray Per-ROI superpixel index arrays. + z_tol : float, optional + Maximum absolute rounded z offset from the median, in um (default 1.5). Returns ------- @@ -428,6 +434,7 @@ def compute_high_res_traces( median_z, unique_motion_ds, mot_inds_to_keep_ds, + z_tol=z_tol, ) ref_d, ref_c, ref_r = ref_pixs_to_drc( @@ -534,6 +541,7 @@ def get_high_res_traces( dmd_pixels_per_row: int, num_channels: int, soma_sps: List[np.ndarray], + z_tol: float = 1.5, ) -> TrialTraceResult: """Read and extract one trial's high-res source traces. @@ -572,6 +580,8 @@ def get_high_res_traces( Number of acquisition channels. soma_sps : list of ndarray Per-ROI superpixel index arrays. + z_tol : float, optional + Maximum absolute rounded z offset from the median, in um (default 1.5). Returns ------- @@ -618,4 +628,5 @@ def get_high_res_traces( dmd_pixels_per_row, num_channels, soma_sps, + z_tol=z_tol, ) diff --git a/src/giant_python/extraction/band/workflow.py b/src/giant_python/extraction/band/workflow.py index 975aa60..866f649 100644 --- a/src/giant_python/extraction/band/workflow.py +++ b/src/giant_python/extraction/band/workflow.py @@ -246,7 +246,7 @@ def _read(trial_ix): ) median_z = np.median(low["lowResMotionZ"]) _, frames_to_keep = motion_binning.select_motion_bins( - unique_motion, mot_inds, low["lowResMotionZ"], z_thresh=params.z_tol + unique_motion, mot_inds, low["lowResMotionZ"], z_tol=params.z_tol ) mean_im = si.compute_mean_image( low_res_data_norm, @@ -348,6 +348,7 @@ def _traces(trial_ix): dmd_pixels_per_row, num_channels, soma_sps, + z_tol=params.z_tol, ) trace_results = map_trials( diff --git a/tests/test_band_workflow.py b/tests/test_band_workflow.py index 04162da..05fb428 100644 --- a/tests/test_band_workflow.py +++ b/tests/test_band_workflow.py @@ -286,6 +286,12 @@ def read_trial( ) ) + bin_trial_motion = stack.enter_context( + patch.object( + traces, "_bin_trial_motion", wraps=traces._bin_trial_motion + ) + ) + input_value = table if loaded else source summary = extract_band_sources( input=input_value, @@ -311,8 +317,11 @@ def read_trial( self.assertEqual(low_read.call_count, 4) self.assertEqual(select_bins.call_count, 2) for call in select_bins.call_args_list: - self.assertEqual(call.kwargs["z_thresh"], params.z_tol) + self.assertEqual(call.kwargs["z_tol"], params.z_tol) self.assertEqual(high_read.call_count, 2) + self.assertEqual(bin_trial_motion.call_count, 2) + for call in bin_trial_motion.call_args_list: + self.assertEqual(call.kwargs["z_tol"], params.z_tol) self.assertEqual(mapper.call_count, 4) for call in mapper.call_args_list: self.assertEqual(call.args[2], options.max_workers) diff --git a/tests/test_bandsilo_background.py b/tests/test_bandsilo_background.py index 5e675e7..ff078c5 100644 --- a/tests/test_bandsilo_background.py +++ b/tests/test_bandsilo_background.py @@ -126,7 +126,7 @@ def test_select_motion_bins_filters_z_and_count(self): mot_inds = np.array([0] * 5 + [1] * 2 + [2] * 5) motion_z = np.zeros(12) keep, frames = motion_binning.select_motion_bins( - unique_motion, mot_inds, motion_z, z_thresh=1.5, min_frames=3 + unique_motion, mot_inds, motion_z, z_tol=1.5, min_frames=3 ) self.assertEqual(list(keep), [0]) self.assertEqual(int(frames.sum()), 5) @@ -139,7 +139,7 @@ def test_z_tolerance_changes_selected_bins(self): for tolerance, expected in ((1.5, [0]), (2.0, [0, 1])): with self.subTest(tolerance=tolerance): keep, frames = motion_binning.select_motion_bins( - unique_motion, mot_inds, motion_z, z_thresh=tolerance + unique_motion, mot_inds, motion_z, z_tol=tolerance ) np.testing.assert_array_equal(keep, expected) np.testing.assert_array_equal(frames, np.isin(mot_inds, expected)) diff --git a/tests/test_bandsilo_traces.py b/tests/test_bandsilo_traces.py index efeddbd..26a54e1 100644 --- a/tests/test_bandsilo_traces.py +++ b/tests/test_bandsilo_traces.py @@ -114,6 +114,29 @@ def _compute(inp): ) +class TestBinTrialMotion(unittest.TestCase): + """The configured z tolerance controls trial-frame retention.""" + + def test_z_tolerance_and_inclusive_boundaries(self): + motion_z = np.array([-3, -2, -1, 0, 1, 2, 3], dtype=float) + args = ( + np.zeros(7), np.zeros(7), motion_z, 0.5, + np.array([[0, 0]]), np.array([0]), + ) + default = tr._bin_trial_motion(*args) + np.testing.assert_array_equal( + default[3], np.abs(motion_z - 0.5) <= 1.5 + ) + for tolerance in (0.0, 0.5, 2.5, 4.0): + with self.subTest(z_tol=tolerance): + _, mot_inds, _, kept = tr._bin_trial_motion( + *args, z_tol=tolerance + ) + expected = np.abs(motion_z - 0.5) <= tolerance + np.testing.assert_array_equal(kept, expected) + np.testing.assert_array_equal(mot_inds[~expected], -1) + + class TestComputeHighResTraces(unittest.TestCase): """compute_high_res_traces per-trial numerics.""" From 819f9a43d29ad55f4acf413273928d99b62186c9 Mon Sep 17 00:00:00 2001 From: m-xie Date: Thu, 1 Oct 2026 00:51:59 +0000 Subject: [PATCH 42/55] conduct nmf on temporally matched filtered signal --- .../extraction/band/localization.py | 25 ++++++++++-- src/giant_python/extraction/band/workflow.py | 4 +- tests/test_bandsilo_nmf.py | 40 ++++++++++++++++++- tests/test_bandsilo_pipeline.py | 11 ++++- 4 files changed, 72 insertions(+), 8 deletions(-) diff --git a/src/giant_python/extraction/band/localization.py b/src/giant_python/extraction/band/localization.py index c23cafa..e843d36 100644 --- a/src/giant_python/extraction/band/localization.py +++ b/src/giant_python/extraction/band/localization.py @@ -26,9 +26,9 @@ This block is intentionally stochastic (random motion order via ``torch.randperm`` in NMF and Adam); the pipeline does not seed the RNG. The ``randperm`` calls are preserved in the same order as the reference so a seeded -run reproduces it exactly. The reference's loss-only ``phi`` recomputation and -``print``/warning diagnostics are dropped (they do not affect the outputs and -consume no RNG). +run with unfiltered input reproduces it exactly. The reference's loss-only +``phi`` recomputation and ``print``/warning diagnostics are dropped (they do not +affect the outputs and consume no RNG). """ from __future__ import annotations @@ -37,6 +37,7 @@ import numpy as np import torch +from scipy import signal from ...progress import progress @@ -665,6 +666,8 @@ def fit_sources( learning_rate: float = 0.01, gd_tol: float = 1e-4, verbose: bool = False, + *, + temporal_kernel: np.ndarray | None = None, ) -> dict: """Localize sources by superpixel-space NMF + Gaussian-profile fitting. @@ -700,6 +703,13 @@ def fit_sources( Adam convergence tolerance. verbose : bool Show a progress bar over the outer NMF/fit iterations when set. + temporal_kernel : ndarray, optional + Temporal matched-filter kernel, normally from ``decay_kernel_1d``. + Convolved directly along each residual row before motion binning, + without compensating for motion differences. Uses zero-padded, + same-length convolution and leaves ``residual`` unchanged. NMF, + low-resolution temporal weights and SNR all use the filtered data. + If omitted, use the unfiltered residual. Returns ------- @@ -712,7 +722,14 @@ def fit_sources( n_motions = unique_motion_to_keep_yx.shape[0] n_frames = residual.shape[1] pixel_coords_tensor = torch.tensor(pixel_coords, dtype=torch.float32) - data_for_nmf = torch.from_numpy(residual.astype(np.float32, copy=False)) + nmf_residual = residual.astype(np.float32, copy=False) + if temporal_kernel is not None: + nmf_residual = signal.convolve( + nmf_residual, + np.asarray(temporal_kernel, dtype=np.float32)[None, :], + mode="same", + ) + data_for_nmf = torch.from_numpy(nmf_residual) num_epochs = n_motions * 5 source_params = init_source_params(source_seeds) diff --git a/src/giant_python/extraction/band/workflow.py b/src/giant_python/extraction/band/workflow.py index 866f649..ef97422 100644 --- a/src/giant_python/extraction/band/workflow.py +++ b/src/giant_python/extraction/band/workflow.py @@ -534,9 +534,10 @@ def _localize( verbose=execution.verbose, ) nan_ct = activity.mask_high_nan_rho(rho) + temporal_kernel = activity.decay_kernel_1d(params.decay_tau_s, align_hz) rho = activity.smooth_rho( rho, - activity.decay_kernel_1d(params.decay_tau_s, align_hz), + temporal_kernel, verbose=execution.verbose, ) act_im = si.accumulate_activity_image( @@ -576,6 +577,7 @@ def _localize( params.d_xy, params.sparse_fac, verbose=execution.verbose, + temporal_kernel=temporal_kernel, ) return ( act_im, diff --git a/tests/test_bandsilo_nmf.py b/tests/test_bandsilo_nmf.py index 72fe30e..4f65fb8 100644 --- a/tests/test_bandsilo_nmf.py +++ b/tests/test_bandsilo_nmf.py @@ -13,7 +13,7 @@ import numpy as np import torch -from giant_python.extraction.band import geometry +from giant_python.extraction.band import activity, geometry from giant_python.extraction.band import localization as nmf from tests.test_bandsilo_background import _small_geometry @@ -257,6 +257,44 @@ def test_deterministic_under_seed(self): a2 = nmf.fit_sources(seeds.copy(), residual, **kwargs)["A"] np.testing.assert_array_equal(a1.numpy(), a2.numpy()) + def test_temporal_filter_matches_direct_row_convolution(self): + """Filter across alternating motions, without changing the residual.""" + g, seeds, residual, mot_yx, coords, n_pix = self._inputs(seed=1) + original = residual.copy() + kernel = activity.decay_kernel_1d(0.1, 10).astype(np.float32) + filtered = np.stack( + [np.convolve(row, kernel, mode="same") for row in residual] + ) + self.assertFalse(np.allclose(filtered, residual)) + kwargs = dict( + h_mots=g["h_mots"], + unique_motion_to_keep_yx=g["umyx"], + mot_inds_yx=mot_yx, + sel_pix_idxs=g["sel_pix_idxs"], + pixel_coords=coords, + n_pixels=n_pix, + d_xy=5, + sparse_fac=float(np.exp(-3.0)), + outer_loop_iters=3, + mult_nmf_max_iters=3, + ) + torch.manual_seed(3) + expected = nmf.fit_sources(seeds.copy(), filtered, **kwargs) + torch.manual_seed(3) + actual = nmf.fit_sources( + seeds.copy(), residual, temporal_kernel=kernel, **kwargs + ) + np.testing.assert_array_equal(residual, original) + self.assertEqual(actual["n_sources"], expected["n_sources"]) + for key in ("A", "phi_low_res", "source_params", "source_snr"): + np.testing.assert_allclose( + actual[key], expected[key], rtol=1e-5, atol=1e-6, + err_msg=key, + ) + np.testing.assert_array_equal( + actual["source_seeds"], expected["source_seeds"] + ) + def test_adam_early_convergence_break(self): """A huge gd_tol triggers the Adam convergence break.""" g, seeds, residual, mot_yx, coords, n_pix = self._inputs(seed=2) diff --git a/tests/test_bandsilo_pipeline.py b/tests/test_bandsilo_pipeline.py index 643d314..e2e46d2 100644 --- a/tests/test_bandsilo_pipeline.py +++ b/tests/test_bandsilo_pipeline.py @@ -144,8 +144,10 @@ def test_dog_settings_and_all_source_columns_reach_output(self): wf.activity, "mask_high_nan_rho", return_value=np.zeros(3) ) ) - stack.enter_context(patch.object(wf.activity, "decay_kernel_1d")) - stack.enter_context( + decay_kernel = stack.enter_context( + patch.object(wf.activity, "decay_kernel_1d") + ) + smooth = stack.enter_context( patch.object(wf.activity, "smooth_rho", return_value=rho) ) stack.enter_context( @@ -195,6 +197,11 @@ def test_dog_settings_and_all_source_columns_reach_output(self): self.assertIs(fit.call_args.args[4], mot_inds) self.assertIs(fit.call_args.args[5], selected) self.assertIs(fit.call_args.args[6], coords) + decay_kernel.assert_called_once_with(BandSiloParams().decay_tau_s, 10) + self.assertIs(smooth.call_args.args[1], decay_kernel.return_value) + self.assertIs( + fit.call_args.kwargs["temporal_kernel"], decay_kernel.return_value + ) self.assertIs(result[2], fitted["A"]) self.assertIs(result[3], fitted["source_params"]) self.assertIs(result[4], fitted["source_snr"]) From f7559ebdad28ed38e2106f52c51daa4fd6a4f223 Mon Sep 17 00:00:00 2001 From: m-xie Date: Thu, 1 Oct 2026 07:37:51 +0000 Subject: [PATCH 43/55] make temporal filter before nmf nan-aware --- .../extraction/band/localization.py | 40 ++++++-- tests/test_bandsilo_nmf.py | 93 +++++++++++++++++-- 2 files changed, 116 insertions(+), 17 deletions(-) diff --git a/src/giant_python/extraction/band/localization.py b/src/giant_python/extraction/band/localization.py index e843d36..95bc892 100644 --- a/src/giant_python/extraction/band/localization.py +++ b/src/giant_python/extraction/band/localization.py @@ -37,9 +37,9 @@ import numpy as np import torch -from scipy import signal from ...progress import progress +from .activity import smooth_rho def sel_pix_gaussian_profile( @@ -705,11 +705,14 @@ def fit_sources( Show a progress bar over the outer NMF/fit iterations when set. temporal_kernel : ndarray, optional Temporal matched-filter kernel, normally from ``decay_kernel_1d``. - Convolved directly along each residual row before motion binning, - without compensating for motion differences. Uses zero-padded, - same-length convolution and leaves ``residual`` unchanged. NMF, + Applied with the activity image's NaN-aware ``smooth_rho`` before + motion binning, without compensating for motion differences. Leaves + ``residual`` unchanged. Frames with insufficient filter coverage in + any row are excluded from NMF and SNR; their temporal weights remain + NaN. Motion bins with no remaining frames are also excluded. NMF, low-resolution temporal weights and SNR all use the filtered data. - If omitted, use the unfiltered residual. + Raises ValueError if no usable frames remain. If omitted, use the + unfiltered residual. Returns ------- @@ -724,11 +727,30 @@ def fit_sources( pixel_coords_tensor = torch.tensor(pixel_coords, dtype=torch.float32) nmf_residual = residual.astype(np.float32, copy=False) if temporal_kernel is not None: - nmf_residual = signal.convolve( - nmf_residual, - np.asarray(temporal_kernel, dtype=np.float32)[None, :], - mode="same", + # smooth_rho operates in place. Dropped frames must not contribute + # even if the caller supplied finite values for them. + nmf_residual = nmf_residual.copy() + nmf_residual[:, mot_inds_yx < 0] = np.nan + nmf_residual = smooth_rho( + nmf_residual, temporal_kernel, verbose=verbose ) + # The shared filter retains NaNs at low-coverage samples. The dense + # NMF solves cannot accept these, so omit affected frames rather + # than interpreting missing observations as zero residuals. + valid_frames = np.all(np.isfinite(nmf_residual), axis=0) + retained = (mot_inds_yx >= 0) & valid_frames + if not np.any(retained): + raise ValueError( + "No usable frames remain after NaN-aware temporal filtering" + ) + kept_motions = np.unique(mot_inds_yx[retained]) + filtered_motion_inds = np.full_like(mot_inds_yx, -1) + filtered_motion_inds[retained] = np.searchsorted( + kept_motions, mot_inds_yx[retained] + ) + mot_inds_yx = filtered_motion_inds + h_mots = [h_mots[i] for i in kept_motions] + n_motions = len(h_mots) data_for_nmf = torch.from_numpy(nmf_residual) num_epochs = n_motions * 5 diff --git a/tests/test_bandsilo_nmf.py b/tests/test_bandsilo_nmf.py index 4f65fb8..851a788 100644 --- a/tests/test_bandsilo_nmf.py +++ b/tests/test_bandsilo_nmf.py @@ -257,19 +257,19 @@ def test_deterministic_under_seed(self): a2 = nmf.fit_sources(seeds.copy(), residual, **kwargs)["A"] np.testing.assert_array_equal(a1.numpy(), a2.numpy()) - def test_temporal_filter_matches_direct_row_convolution(self): - """Filter across alternating motions, without changing the residual.""" + def test_temporal_filter_matches_activity_filter(self): + """Reuse activity smoothing and omit its low-coverage edge frames.""" g, seeds, residual, mot_yx, coords, n_pix = self._inputs(seed=1) original = residual.copy() kernel = activity.decay_kernel_1d(0.1, 10).astype(np.float32) - filtered = np.stack( - [np.convolve(row, kernel, mode="same") for row in residual] - ) + filtered = activity.smooth_rho(residual.copy(), kernel) + filtered_motions = mot_yx.copy() + filtered_motions[~np.all(np.isfinite(filtered), axis=0)] = -1 + self.assertTrue(np.any(filtered_motions < 0)) self.assertFalse(np.allclose(filtered, residual)) kwargs = dict( h_mots=g["h_mots"], unique_motion_to_keep_yx=g["umyx"], - mot_inds_yx=mot_yx, sel_pix_idxs=g["sel_pix_idxs"], pixel_coords=coords, n_pixels=n_pix, @@ -279,10 +279,13 @@ def test_temporal_filter_matches_direct_row_convolution(self): mult_nmf_max_iters=3, ) torch.manual_seed(3) - expected = nmf.fit_sources(seeds.copy(), filtered, **kwargs) + expected = nmf.fit_sources( + seeds.copy(), filtered, mot_inds_yx=filtered_motions, **kwargs + ) torch.manual_seed(3) actual = nmf.fit_sources( - seeds.copy(), residual, temporal_kernel=kernel, **kwargs + seeds.copy(), residual, mot_inds_yx=mot_yx, + temporal_kernel=kernel, **kwargs ) np.testing.assert_array_equal(residual, original) self.assertEqual(actual["n_sources"], expected["n_sources"]) @@ -295,6 +298,80 @@ def test_temporal_filter_matches_direct_row_convolution(self): actual["source_seeds"], expected["source_seeds"] ) + def test_temporal_filter_dropped_frames_survive_pruning(self): + """Missing samples must not poison otherwise usable source fits.""" + g, seeds, residual, mot_yx, coords, n_pix = self._inputs() + residual = np.tile(residual, (1, 16)) + mot_yx = np.tile(mot_yx, 16) + mot_yx[128] = -1 + original_motions = mot_yx.copy() + kernel = activity.decay_kernel_1d(0.15, 100) + for missing_value in (np.nan, 1e6): + with self.subTest(missing_value=missing_value): + residual[:, 128] = missing_value + original = residual.copy() + torch.manual_seed(7) + out = nmf.fit_sources( + seeds.copy(), residual, g["h_mots"], g["umyx"], + mot_yx, g["sel_pix_idxs"], coords, n_pix, + d_xy=5, sparse_fac=float(np.exp(-3.0)), + outer_loop_iters=3, mult_nmf_max_iters=4, + temporal_kernel=kernel, + ) + self.assertGreater(out["n_sources"], 0) + self.assertTrue(np.all(np.isfinite(out["source_snr"]))) + self.assertTrue(torch.all(torch.isfinite(out["A"]))) + self.assertTrue(torch.all(torch.isnan(out["phi_low_res"][128]))) + self.assertTrue(torch.any(torch.isfinite(out["phi_low_res"]))) + np.testing.assert_array_equal(residual, original) + np.testing.assert_array_equal(mot_yx, original_motions) + + def test_temporal_filter_excludes_empty_motion_bins(self): + """Low coverage can remove an entire bin without poisoning Adam.""" + g, seeds, residual, mot_yx, coords, n_pix = self._inputs() + # Only the first frame uses bin 0; a uniform kernel rejects both + # edges. An isolated row-wise NaN also rejects nearby frames. + mot_yx[:] = 1 + mot_yx[0] = 0 + residual[0, 8] = np.nan + kernel = np.ones(3) / 3 + filtered = activity.smooth_rho(residual.copy(), kernel) + expected_motions = np.full_like(mot_yx, -1) + expected_motions[np.all(np.isfinite(filtered), axis=0)] = 0 + kwargs = dict( + sel_pix_idxs=g["sel_pix_idxs"], pixel_coords=coords, + n_pixels=n_pix, d_xy=5, sparse_fac=0.05, + outer_loop_iters=1, mult_nmf_max_iters=3, + ) + torch.manual_seed(7) + expected = nmf.fit_sources( + seeds.copy(), filtered, [g["h_mots"][1]], g["umyx"][1:], + expected_motions, **kwargs, + ) + torch.manual_seed(7) + actual = nmf.fit_sources( + seeds.copy(), residual, g["h_mots"], g["umyx"], mot_yx, + temporal_kernel=kernel, **kwargs, + ) + self.assertTrue(torch.all(torch.isfinite(actual["A"]))) + for key in ("A", "phi_low_res", "source_params", "source_snr"): + np.testing.assert_allclose(actual[key], expected[key]) + excluded = expected_motions < 0 + self.assertTrue( + torch.all(torch.isnan(actual["phi_low_res"][excluded])) + ) + + def test_temporal_filter_no_usable_frames(self): + """Report missing data explicitly instead of silently pruning all seeds.""" + g, seeds, residual, mot_yx, coords, n_pix = self._inputs() + residual[:] = np.nan + with self.assertRaisesRegex(ValueError, "No usable frames remain"): + nmf.fit_sources( + seeds, residual, g["h_mots"], g["umyx"], mot_yx, + g["sel_pix_idxs"], coords, n_pix, d_xy=5, + sparse_fac=0.05, temporal_kernel=np.ones(1), + ) + def test_adam_early_convergence_break(self): """A huge gd_tol triggers the Adam convergence break.""" g, seeds, residual, mot_yx, coords, n_pix = self._inputs(seed=2) From fe6473970521101013da73d4cb1b8c26895068c3 Mon Sep 17 00:00:00 2001 From: m-xie Date: Thu, 1 Oct 2026 19:48:51 +0000 Subject: [PATCH 44/55] add option to do simple weighted average trace extraction --- README.md | 18 +++ src/giant_python/cli.py | 10 +- src/giant_python/extraction/band/traces.py | 52 ++++++-- src/giant_python/extraction/band/workflow.py | 2 + src/giant_python/gui/parameters_gui.py | 7 + src/giant_python/models/experiment_summary.py | 5 +- src/giant_python/models/params.py | 7 + src/giant_python/numerics/baseline.py | 7 +- tests/test_band_workflow.py | 33 ++++- tests/test_bandsilo_traces.py | 120 +++++++++++++++++- tests/test_cli.py | 3 + tests/test_configuration.py | 12 +- tests/test_parameter_gui.py | 14 +- 13 files changed, 270 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index bb0a7bb..60ec751 100644 --- a/README.md +++ b/README.md @@ -131,6 +131,24 @@ For data not collected on SLAP2, the reference GIAnT-MATLAB pipeline sets Epochs ## Pipeline Outputs +### Optional weighted-average traces + +Set `BandSiloParams(simple_trace_extraction=True)` (or the equivalent parameter +dictionary) to extract final traces by spatial-profile weighted averaging +instead of least squares. The CLI option is `--simple-trace-extraction`; the +parameter GUI also provides a checkbox. The default is `False`. + +Source identification and profile fitting are unchanged. For each motion bin, +the fitted profile is projected into superpixel space and normalized by its +sum. The normalized intensities average the background-subtracted superpixel +activity into `dF_ls` and the background into `F0`, preserving their compatible +scales for subsequent baseline processing. Unlike least squares, this does not +de-mix overlapping sources. Unobserved frames and profiles with zero observed +support yield NaN. The output schema retains `dF_ls` in both modes, and summary +parameters record `simple_trace_extraction`. + +### Output schema + The extensive trees and field tables below are **reference schemas**, not a list of everything Python emits. The working BandSILo writer emits a subset of the experiment summary: source `profiles`, `coords`, `dF_ls`, `F0`, `SNR`; visualizations; `Z_depths`; global `F`; frame bookkeeping and shifts; and, when enabled, ROI `labels`, `mask`, and `F`. Disabled annotations omit the ROI group; enabled-but-empty annotations are distinct. **Future/reference-only output:** `dF_denoised`, `events`, ROI `Fsvd`, and the entire per-trial summary are **not emitted**. Fields retained on model classes do not imply that the band codec reads/writes them. Arbitrary reference files are not guaranteed to round-trip losslessly through `ExperimentSummary`. diff --git a/src/giant_python/cli.py b/src/giant_python/cli.py index f7e4e3f..014e87e 100644 --- a/src/giant_python/cli.py +++ b/src/giant_python/cli.py @@ -26,6 +26,11 @@ def _add_silo_options(parser: argparse.ArgumentParser) -> None: """Attach the shared source-extraction options to a subparser.""" + parser.add_argument( + "--simple-trace-extraction", + action="store_true", + help="Use profile-weighted averages instead of least squares for final traces.", + ) parser.add_argument( "--z-tol", type=float, @@ -114,7 +119,10 @@ def _options_from_args( ) -> tuple[BandSiloParams, ExecutionOptions, AnnotationOptions]: """Build independent science, execution and annotation configuration.""" return ( - BandSiloParams(z_tol=args.z_tol), + BandSiloParams( + z_tol=args.z_tol, + simple_trace_extraction=args.simple_trace_extraction, + ), ExecutionOptions( max_workers=args.max_workers, verbose=args.verbose, diff --git a/src/giant_python/extraction/band/traces.py b/src/giant_python/extraction/band/traces.py index f351b20..88b8bd8 100644 --- a/src/giant_python/extraction/band/traces.py +++ b/src/giant_python/extraction/band/traces.py @@ -5,9 +5,10 @@ the high-res superpixel activity, interpolates the alignment motion/background onto the trial's downsample grid, bins the frames by motion (matching the low-res motion bins used during NMF), and — per motion bin — projects the fixed -spatial profiles ``A_final`` into superpixel space and least-squares-solves the -per-source temporal traces (``phi``, the least-squares dF) and the background -projection (``F0``). It also computes the global and per-user-ROI fluorescence. +spatial profiles ``A_final`` into superpixel space and extracts per-source +temporal traces (``phi``, stored as ``dF_ls``) and background (``F0``) via +least squares or normalized profile-weighted averages. It also computes the +global and per-user-ROI fluorescence. The pure compute (:func:`compute_high_res_traces`) is factored out of the IO wrapper (:func:`get_high_res_traces`, which reads the SLAP2 file lazily via @@ -189,6 +190,22 @@ def _bin_trial_motion( return unique_motion, mot_inds, mot_inds_to_keep, frames_to_keep +def _weighted_trace_average( + x: torch.Tensor, data: torch.Tensor +) -> torch.Tensor: + """Average superpixels using each projected source profile's intensities. + + ``x`` is (superpixels, sources), ``data`` is (superpixels, frames), and + the result is (frames, sources). Normalize in superpixel space for each + motion bin: projection through H need not preserve the profile's mass. + Sources with no observed profile support return NaN, not a zero signal. + Unlike least squares, overlapping sources are not de-mixed. + """ + mass = x.sum(dim=0) + weights = x / mass.masked_fill(mass <= 0, float("nan")) + return (weights.T @ data).T + + def _solve_trial_phi_f0( data: np.ndarray, background: np.ndarray, @@ -201,8 +218,9 @@ def _solve_trial_phi_f0( a_final: torch.Tensor, num_super_pixels: int, dmd_pixels_per_row: int, + simple_trace_extraction: bool = False, ) -> Tuple[torch.Tensor, torch.Tensor]: - """Least-squares-solve per-source traces and background projection. + """Extract per-source traces and background projection. For each kept motion bin, shifts ``H`` by the bin's motion, projects ``A_final`` into superpixel space, and solves for the source temporal @@ -231,12 +249,15 @@ def _solve_trial_phi_f0( Superpixel count (``H`` row count). dmd_pixels_per_row : int Grid geometry (column-shift stride). + simple_trace_extraction : bool, optional + Use normalized profile-intensity weighted averages instead of least + squares for both dF and F0 (default False). Returns ------- phi, f0 : torch.Tensor of shape (n_frames, n_sources) - Per-source least-squares dF and background projection (NaN where no - motion bin applies). + Per-source dF and background projection (NaN where no motion bin + applies, or where weighted averaging has no profile support). """ n_sources = a_final.shape[1] n_frames = data.shape[1] @@ -255,15 +276,18 @@ def _solve_trial_phi_f0( dmd_pixels_per_row, ) + extract = ( + _weighted_trace_average if simple_trace_extraction else solve_phi_motion + ) for motion_idx in mot_inds_to_keep: motion_frames = np.flatnonzero(mot_inds == motion_idx) x = torch.sparse.mm(h_mots[motion_idx], a_final[sel_pix_idxs, :]) # phi from the background-subtracted residual; F0 from the background, - # both via the shared regularized normal-equations solve. - phi[motion_frames, :] = solve_phi_motion( + # both using the same extraction method and spatial profiles. + phi[motion_frames, :] = extract( x, torch.from_numpy(residual[:, motion_frames].astype(np.float32)) ) - f0[motion_frames, :] = solve_phi_motion( + f0[motion_frames, :] = extract( x, torch.from_numpy(background[:, motion_frames].astype(np.float32)), ) @@ -366,6 +390,7 @@ def compute_high_res_traces( num_channels: int, soma_sps: List[np.ndarray], z_tol: float = 1.5, + simple_trace_extraction: bool = False, ) -> TrialTraceResult: """Compute one trial's high-res source traces from loaded arrays. @@ -404,6 +429,9 @@ def compute_high_res_traces( Per-ROI superpixel index arrays. z_tol : float, optional Maximum absolute rounded z offset from the median, in um (default 1.5). + simple_trace_extraction : bool, optional + Average superpixels weighted by normalized source profiles instead + of least-squares fitting final dF and F0 traces (default False). Returns ------- @@ -463,6 +491,7 @@ def compute_high_res_traces( a_final, num_super_pixels, dmd_pixels_per_row, + simple_trace_extraction=simple_trace_extraction, ) global_f = _compute_global_f(data, data2, frames_to_keep, num_channels) @@ -542,6 +571,7 @@ def get_high_res_traces( num_channels: int, soma_sps: List[np.ndarray], z_tol: float = 1.5, + simple_trace_extraction: bool = False, ) -> TrialTraceResult: """Read and extract one trial's high-res source traces. @@ -582,6 +612,9 @@ def get_high_res_traces( Per-ROI superpixel index arrays. z_tol : float, optional Maximum absolute rounded z offset from the median, in um (default 1.5). + simple_trace_extraction : bool, optional + Average superpixels weighted by normalized source profiles instead + of least-squares fitting final dF and F0 traces (default False). Returns ------- @@ -629,4 +662,5 @@ def get_high_res_traces( num_channels, soma_sps, z_tol=z_tol, + simple_trace_extraction=simple_trace_extraction, ) diff --git a/src/giant_python/extraction/band/workflow.py b/src/giant_python/extraction/band/workflow.py index ef97422..4deab28 100644 --- a/src/giant_python/extraction/band/workflow.py +++ b/src/giant_python/extraction/band/workflow.py @@ -160,6 +160,7 @@ def _params_dict( "vif": params.vif, "peakth": params.peakth, "peak_buffer": params.peak_buffer, + "simple_trace_extraction": params.simple_trace_extraction, "draw_user_rois": annotations.enabled, } @@ -349,6 +350,7 @@ def _traces(trial_ix): num_channels, soma_sps, z_tol=params.z_tol, + simple_trace_extraction=params.simple_trace_extraction, ) trace_results = map_trials( diff --git a/src/giant_python/gui/parameters_gui.py b/src/giant_python/gui/parameters_gui.py index 79faefc..d5c9cfc 100644 --- a/src/giant_python/gui/parameters_gui.py +++ b/src/giant_python/gui/parameters_gui.py @@ -65,6 +65,12 @@ def run_parameter_gui( sparse_var = tk.StringVar(value=str(float(np.log(base.sparse_fac)))) ttk.Entry(frame, textvariable=sparse_var, width=15).grid(row=row, column=1) row += 1 + ttk.Label(frame, text="Simple Trace Extraction?").grid( + row=row, column=0, sticky=tk.W + ) + simple_trace_var = tk.BooleanVar(value=base.simple_trace_extraction) + ttk.Checkbutton(frame, variable=simple_trace_var).grid(row=row, column=1) + row += 1 ttk.Label(frame, text="Draw User ROIs?").grid( row=row, column=0, sticky=tk.W ) @@ -90,6 +96,7 @@ def on_ok(): "peak_buffer": int(entries["peak_buffer"].get()), "z_tol": float(entries["z_tol"].get()), "sparse_fac": float(np.exp(float(sparse_var.get()))), + "simple_trace_extraction": bool(simple_trace_var.get()), } result["options"] = ( replace(base, **updates), diff --git a/src/giant_python/models/experiment_summary.py b/src/giant_python/models/experiment_summary.py index 8eb01ef..1b8f6de 100644 --- a/src/giant_python/models/experiment_summary.py +++ b/src/giant_python/models/experiment_summary.py @@ -22,8 +22,9 @@ class Source: coords : ndarray or None 0-indexed ``[z, y, x]`` location. df_ls, df_denoised, events : ndarray or None - Least-squares dF, denoised dF, and deconvolved events, - shaped ``(channels, frames)``. + Extracted dF (least squares or profile-weighted average, after baseline + re-estimation), denoised dF, and deconvolved events, shaped + ``(channels, frames)``. The historical ``df_ls`` name is retained. f0 : ndarray or None Per-channel baseline. snr : float or None diff --git a/src/giant_python/models/params.py b/src/giant_python/models/params.py index 7c05cba..fd6070d 100644 --- a/src/giant_python/models/params.py +++ b/src/giant_python/models/params.py @@ -56,6 +56,10 @@ class BandSiloParams: z_tol : float Positive z tolerance in microns: maximum absolute motion-bin z deviation from the median frame z for low-resolution frame selection. + simple_trace_extraction : bool + Use normalized spatial-profile weighted averages of superpixels for + final dF and background traces instead of least squares. Source + localization is unchanged. Defaults to False. Notes ----- @@ -77,6 +81,7 @@ class BandSiloParams: peak_buffer: int = 3 psf_dilation: int = 17 z_tol: float = 1.5 + simple_trace_extraction: bool = False def __post_init__(self) -> None: """Validate values supported by the existing numerical path.""" @@ -93,6 +98,8 @@ def __post_init__(self) -> None: _positive(name, getattr(self, name)) for name in ("d_xy", "peak_buffer", "psf_dilation"): _positive(name, getattr(self, name), integer=True) + if not isinstance(self.simple_trace_extraction, bool): + raise ValueError("simple_trace_extraction must be a bool") @dataclass diff --git a/src/giant_python/numerics/baseline.py b/src/giant_python/numerics/baseline.py index d63648d..04ba9d7 100644 --- a/src/giant_python/numerics/baseline.py +++ b/src/giant_python/numerics/baseline.py @@ -166,16 +166,17 @@ def assemble_dff( """Assemble ``F``, re-estimate ``F0``, and form ``dF``/``dFF``. Reconstructs total fluorescence ``F = dF_ls + F0_ls`` from the per-motion - least-squares source and background traces, re-estimates the baseline with + source and background traces (least squares or weighted averages), + re-estimates the baseline with :func:`compute_f0`, and computes ``dF = F - F0`` and ``dFF = dF / clip(F0, 1e-4, inf)``. Parameters ---------- d_f_ls : ndarray of shape (T, n_sources) - Concatenated per-source least-squares dF traces (``phi``). + Concatenated per-source dF traces (``phi``). f0_ls : ndarray of shape (T, n_sources) - Concatenated per-source least-squares background traces. + Concatenated per-source background traces in the same extraction scale. denoise_window, hull_window : int Windows passed to :func:`compute_f0`. diff --git a/tests/test_band_workflow.py b/tests/test_band_workflow.py index 05fb428..62e7d43 100644 --- a/tests/test_band_workflow.py +++ b/tests/test_band_workflow.py @@ -31,7 +31,7 @@ class TestBandWorkflow(unittest.TestCase): """Exercise the public API through the driver to saved results.""" - def _run_mock_session(self, *, loaded=False, rois=False): + def _run_mock_session(self, *, loaded=False, rois=False, simple=False): """Check schema, policy and motion values in a two-path session.""" n_frames, side, n_sp = 120, 15, 49 positions = [(r, c) for r in range(4, 11) for c in range(4, 11)] @@ -49,6 +49,7 @@ def _run_mock_session(self, *, loaded=False, rois=False): science_args = dict( analyze_hz=10, denoise_window_s=0.3, baseline_window_s=0.5, z_tol=2.5, + simple_trace_extraction=simple, ) params = BandSiloParams(**science_args) before = (asdict(params), asdict(options), asdict(annotation)) @@ -345,6 +346,8 @@ def read_trial( assert summary.params is not None assert restored.params is not None self.assertEqual(summary.params["numChannels"], 2) + self.assertEqual(summary.params["simple_trace_extraction"], simple) + self.assertEqual(restored.params["simple_trace_extraction"], simple) self.assertNotIn("operator", summary.params) self.assertNotIn("operator", restored.params) self.assertEqual(summary.params["draw_user_rois"], rois) @@ -385,6 +388,27 @@ def read_trial( saved.sources[0].df_ls, equal_nan=True, ) + if simple: + # Constant activity: averaging must preserve its scale, + # regardless of the projected profile's non-unit mass. + np.testing.assert_allclose( + runtime_result.trace_results[0].d_f, + 0.25 * (10 + 5 * dmd_ix), rtol=1e-6, + ) + np.testing.assert_allclose( + runtime_result.trace_results[0].f0_ls, + 0.75 * (10 + 5 * dmd_ix), rtol=1e-6, + ) + np.testing.assert_allclose( + saved.sources[0].df_ls[0], + 0, atol=1e-6, + ) + # Existing baseline re-estimation removes a constant + # signal but preserves total weighted fluorescence. + np.testing.assert_allclose( + saved.sources[0].df_ls[0] + saved.sources[0].f0[0], + 10 + 5 * dmd_ix, rtol=1e-6, + ) self.assertEqual(len(path.user_rois), int(rois)) self.assertEqual(saved.annotation_enabled, rois) if rois: @@ -407,6 +431,13 @@ def test_loaded_table_is_not_reopened(self): """Prepare a loaded model directly without a second file read.""" self._run_mock_session(loaded=True) + def test_simple_extraction_reaches_saved_traces(self): + """Opt-in averaging reaches per-trial extraction and HDF5 output.""" + with patch.object( + traces, "solve_phi_motion", side_effect=AssertionError("LS called") + ): + self._run_mock_session(loaded=True, simple=True) + def test_split_policy_preserves_rois_and_caller_options(self): """Split policy leaves scientific options unchanged.""" self._run_mock_session(loaded=True, rois=True) diff --git a/tests/test_bandsilo_traces.py b/tests/test_bandsilo_traces.py index 26a54e1..2d9e2a4 100644 --- a/tests/test_bandsilo_traces.py +++ b/tests/test_bandsilo_traces.py @@ -8,6 +8,7 @@ """ import unittest +from unittest.mock import patch import numpy as np import torch @@ -88,7 +89,7 @@ def _trial_inputs(num_channels=1, unique_motion_ds=None, seed=2): ) -def _compute(inp): +def _compute(inp, **kwargs): """Call compute_high_res_traces with the assembled inputs dict.""" g = inp["g"] return tr.compute_high_res_traces( @@ -111,9 +112,86 @@ def _compute(inp): g["npr"], inp["num_channels"], inp["soma_sps"], + **kwargs, ) +class TestWeightedTraceAverage(unittest.TestCase): + """Simple traces use profile intensities, not a regression coefficient.""" + + def test_normalized_intensities_and_scale_invariance(self): + """Overlapping and duplicate profiles are averaged independently.""" + profiles = torch.tensor([[1., 0., 7.], [3., 2., 21.], [0., 2., 0.]]) + data = torch.tensor([[2., -2.], [6., 4.], [10., 8.]]) + expected = np.array([[5., 8., 5.], [2.5, 6., 2.5]]) + actual = tr._weighted_trace_average(profiles, data) + np.testing.assert_allclose(actual, expected) + np.testing.assert_allclose( + tr._weighted_trace_average(profiles * 0.01, data), + expected, rtol=1e-6, + ) + + def test_uniform_zero_and_empty_profiles(self): + """Uniform weights give the mean; no support is NaN, not zero.""" + data = torch.tensor([[2., 4.], [6., 8.]]) + profiles = torch.tensor([[3., 0.], [3., 0.]]) + result = tr._weighted_trace_average(profiles, data) + np.testing.assert_allclose(result[:, 0], [4., 6.]) + self.assertTrue(torch.isnan(result[:, 1]).all()) + self.assertEqual( + tr._weighted_trace_average(torch.empty(2, 0), data).shape, + (2, 0), + ) + + def test_motion_projection_background_and_dropped_frames(self): + """Normalize after motion projection and use identical dF/F0 weights.""" + profiles = torch.tensor([[1., 2.], [3., 0.], [0., 4.]]) + operators_by_motion = [ + torch.eye(3).to_sparse(), + torch.tensor([[0., 2., 0.], [0., 0., 1.], [0., 0., 0.]]).to_sparse(), + ] + data = np.array([[2., 4., 8., 1.], [6., 8., 2., 2.], + [10., 12., 4., 3.]], dtype=np.float32) + background = np.array([[1., 1., 2., 0.], [2., 2., 1., 0.], + [3., 3., 3., 0.]], dtype=np.float32) + bins = np.array([0, 0, 1, -1]) + args = ( + data, background, np.array([[0, 0], [1, 0]]), bins, + np.array([0, 1]), np.empty((0, 2)), np.empty(0), np.arange(3), + profiles, 3, 3, + ) + with patch.object( + tr, "build_motion_h_matrices", return_value=operators_by_motion + ), patch.object( + tr, "solve_phi_motion", side_effect=AssertionError("LS called") + ): + phi, f0 = tr._solve_trial_phi_f0( + *args, simple_trace_extraction=True + ) + for motion, h in enumerate(operators_by_motion): + selected = np.flatnonzero(bins == motion) + projected = (h.to_dense() @ profiles).numpy() + for source in range(profiles.shape[1]): + weights = projected[:, source] + np.testing.assert_allclose( + phi[selected, source], + np.average((data - background)[:, selected], axis=0, + weights=weights), rtol=1e-6, + ) + np.testing.assert_allclose( + f0[selected, source], + np.average(background[:, selected], axis=0, + weights=weights), rtol=1e-6, + ) + np.testing.assert_allclose( + (phi + f0)[selected, source], + np.average(data[:, selected], axis=0, weights=weights), + rtol=1e-6, + ) + self.assertTrue(torch.isnan(phi[-1]).all()) + self.assertTrue(torch.isnan(f0[-1]).all()) + + class TestBinTrialMotion(unittest.TestCase): """The configured z tolerance controls trial-frame retention.""" @@ -140,6 +218,37 @@ def test_z_tolerance_and_inclusive_boundaries(self): class TestComputeHighResTraces(unittest.TestCase): """compute_high_res_traces per-trial numerics.""" + def test_default_remains_least_squares(self): + """Omitting the flag retains the existing solver for every bin.""" + inp = _trial_inputs() + with patch.object(tr, "solve_phi_motion", wraps=tr.solve_phi_motion) as ls: + default = _compute(inp) + self.assertGreater(ls.call_count, 0) + explicit = _compute(inp, simple_trace_extraction=False) + np.testing.assert_array_equal(default.d_f, explicit.d_f) + np.testing.assert_array_equal(default.f0_ls, explicit.f0_ls) + + def test_simple_mode_preserves_mask_and_auxiliary_outputs(self): + """Only dF/F0 extraction changes; z rejection and other channels stay.""" + inp = _trial_inputs(num_channels=2) + inp["a_data"]["motionDSz"][20:40] = 5 + least_squares = _compute(inp) + with patch.object( + tr, "solve_phi_motion", side_effect=AssertionError("LS called") + ): + simple = _compute(inp, simple_trace_extraction=True) + self.assertTrue(np.isnan(simple.d_f).any()) + self.assertTrue(np.isfinite(simple.d_f).any()) + for a, b in ((simple.d_f, least_squares.d_f), + (simple.f0_ls, least_squares.f0_ls)): + self.assertEqual(a.shape, b.shape) + np.testing.assert_array_equal(np.isnan(a), np.isnan(b)) + for field in ("global_f", "user_roi_f", "frame_line_idxs", + "selected_pixels", "reference_offsets", "online_shifts"): + np.testing.assert_array_equal( + getattr(simple, field), getattr(least_squares, field) + ) + def test_single_channel_shapes_and_fit(self): """Returns the 8-tuple; phi/F0 are per-frame per-source with fits.""" inp = _trial_inputs(num_channels=1) @@ -223,6 +332,12 @@ def test_skipped_trial_returns_empty(self): def test_kept_trial_reads_and_computes(self): """A kept trial reads (monkeypatched) then computes the traces.""" + for simple in (False, True): + with self.subTest(simple_trace_extraction=simple): + self._check_kept_trial(simple) + + def _check_kept_trial(self, simple): + """Verify IO wrapper forwards the selected extraction mode.""" inp = _trial_inputs(num_channels=1) g = inp["g"] @@ -259,12 +374,13 @@ def fake_load(*args, **kwargs): g["npr"], 1, inp["soma_sps"], + simple_trace_extraction=simple, ) finally: tr._load_high_res_trial_data = orig # matches a direct compute with the same inputs - expected = _compute(inp) + expected = _compute(inp, simple_trace_extraction=simple) np.testing.assert_allclose( np.nan_to_num(out[0]), np.nan_to_num(expected[0]), rtol=1e-6 ) diff --git a/tests/test_cli.py b/tests/test_cli.py index 650b533..daed18d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -113,6 +113,7 @@ def test_maps_fields(self): params, execution, annotations = cli._options_from_args(args) self.assertEqual(params.peakth, 7.0) self.assertEqual(params.z_tol, 1.5) + self.assertFalse(params.simple_trace_extraction) self.assertFalse(hasattr(params, "scan_mode")) self.assertTrue(annotations.enabled) self.assertTrue(annotations.interactive) @@ -209,11 +210,13 @@ def test_extract_service(self): "2.5", "--headless", "--draw-user-rois", + "--simple-trace-extraction", ] ) self.assertEqual(code, 0) self.assertEqual(extract.call_args.args[0], "curated_v2.h5") self.assertEqual(extract.call_args.args[1].z_tol, 2.5) + self.assertTrue(extract.call_args.args[1].simple_trace_extraction) self.assertEqual(extract.call_args.kwargs["scan_mode"], "band") self.assertEqual(extract.call_args.kwargs["execution"].max_trials, 4) self.assertTrue(extract.call_args.kwargs["annotations"].enabled) diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 503b5d2..4b333ac 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -22,6 +22,7 @@ def test_defaults_match_existing_science(self): self.assertEqual(science.denoise_window_s, 1.0) self.assertEqual(science.vif, 1.38) self.assertEqual(science.z_tol, 1.5) + self.assertFalse(science.simple_trace_extraction) self.assertEqual(science.sparse_fac, math.exp(-3.0)) self.assertEqual(execution, ExecutionOptions(6, False, None)) self.assertEqual(annotations, AnnotationOptions(False, None)) @@ -63,7 +64,8 @@ def test_every_result_is_fresh(self): def test_dictionary_inputs_and_overrides_are_not_modified(self): """Each dictionary affects only its scope and is copied.""" - params = {"analyze_hz": 80.0, "z_tol": 2.5} + params = {"analyze_hz": 80.0, "z_tol": 2.5, + "simple_trace_extraction": True} execution = {"max_workers": 2, "verbose": True} annotations = { "enabled": True, @@ -75,6 +77,7 @@ def test_dictionary_inputs_and_overrides_are_not_modified(self): ) self.assertEqual(science.analyze_hz, 80.0) self.assertEqual(science.z_tol, 2.5) + self.assertTrue(science.simple_trace_extraction) self.assertEqual(run, ExecutionOptions(2, True, None)) self.assertEqual(roi, AnnotationOptions(True, False)) self.assertEqual((params, execution, annotations), original) @@ -83,6 +86,13 @@ def test_dictionary_inputs_and_overrides_are_not_modified(self): class TestConfigurationValidation(unittest.TestCase): """Reject unsupported settings before they reach numerical kernels.""" + def test_simple_trace_extraction_requires_bool(self): + """Strings and integers cannot silently select an extraction mode.""" + for value in (0, 1, "true", "false", None): + with self.subTest(value=value): + with self.assertRaisesRegex(ValueError, "simple_trace_extraction"): + resolve_band_options({"simple_trace_extraction": value}) + def test_positive_scientific_values(self): """Rates, windows and positive scale values must be finite.""" for name in ( diff --git a/tests/test_parameter_gui.py b/tests/test_parameter_gui.py index 4595880..d9881e2 100644 --- a/tests/test_parameter_gui.py +++ b/tests/test_parameter_gui.py @@ -21,6 +21,7 @@ def setUp(self): self.tk = MagicMock() self.root = self.tk.Tk.return_value self.variables = [] + self.boolean_variables = [] self.buttons = {} self.tk.StringVar.side_effect = self._string_var self.tk.BooleanVar.side_effect = self._boolean_var @@ -37,9 +38,10 @@ def _string_var(self, *, value): return variable def _boolean_var(self, *, value): - """Track the ROI inclusion control separately from text fields.""" + """Track checkbox controls separately from text fields.""" self.roi_var = MagicMock() self.roi_var.get.return_value = value + self.boolean_variables.append(self.roi_var) return self.roi_var def _button(self, parent, *, text, command): @@ -59,6 +61,7 @@ def interact(): self.variables[8].get.return_value = "3" self.variables[9].get.return_value = "2.5" self.variables[10].get.return_value = "-2" + self.boolean_variables[0].get.return_value = True self.roi_var.get.return_value = True self.buttons["OK"]() @@ -69,6 +72,8 @@ def interact(): self.assertEqual(science.analyze_hz, 125) self.assertEqual(science.peakth, 8) self.assertEqual(science.z_tol, 2.5) + self.assertTrue(science.simple_trace_extraction) + self.assertFalse(params.simple_trace_extraction) self.assertEqual(params.z_tol, 1.5) self.assertAlmostEqual(science.sparse_fac, math.exp(-2)) labels = [ @@ -92,6 +97,13 @@ def test_cancel_returns_none(self): self.assertIsNone(run_parameter_gui()) self.root.destroy.assert_called_once() + def test_simple_trace_checkbox_preserves_initial_value(self): + """An enabled mode survives accepting the form without edits.""" + self.root.mainloop.side_effect = lambda: self.buttons["OK"]() + result = run_parameter_gui(BandSiloParams(simple_trace_extraction=True)) + self.assertTrue(result[0].simple_trace_extraction) + self.assertFalse(result[2].enabled) + def test_invalid_input_shows_error_without_accepting(self): """Invalid scientific values leave the form open for correction.""" From a610ff02c20bcf009bd7592da9e7e3eda2f7f5e0 Mon Sep 17 00:00:00 2001 From: m-xie Date: Fri, 2 Oct 2026 06:06:17 +0000 Subject: [PATCH 45/55] replace least squares solvel with mldivide-style algorithm --- .../extraction/band/localization.py | 42 +++++-- tests/test_bandsilo_nmf.py | 118 +++++++++++++++++- 2 files changed, 146 insertions(+), 14 deletions(-) diff --git a/src/giant_python/extraction/band/localization.py b/src/giant_python/extraction/band/localization.py index 95bc892..73083c4 100644 --- a/src/giant_python/extraction/band/localization.py +++ b/src/giant_python/extraction/band/localization.py @@ -25,10 +25,10 @@ This block is intentionally stochastic (random motion order via ``torch.randperm`` in NMF and Adam); the pipeline does not seed the RNG. The -``randperm`` calls are preserved in the same order as the reference so a seeded -run with unfiltered input reproduces it exactly. The reference's loss-only -``phi`` recomputation and ``print``/warning diagnostics are dropped (they do not -affect the outputs and consume no RNG). +``randperm`` calls are preserved in the same order as the reference, but the +unregularized QR temporal solve changes numerical results. The reference's +loss-only ``phi`` recomputation and ``print``/warning diagnostics are dropped +(they do not affect the outputs and consume no RNG). """ from __future__ import annotations @@ -37,6 +37,7 @@ import numpy as np import torch +from scipy.linalg import qr, solve_triangular from ...progress import progress from .activity import smooth_rho @@ -271,15 +272,36 @@ def solve_phi_motion( ) -> torch.Tensor: """Least-squares temporal weights for one motion's superpixel profiles. - Solves the regularized normal equations - ``(XᵀX + 1e-10 I) phi = Xᵀ data`` and returns ``phiᵀ`` (frames x sources). + Uses column-pivoted QR, ``X[:, piv] = Q R``, to return a basic solution + as in MATLAB's rectangular ``mldivide``: solve the leading independent + triangular block and set the remaining pivoted coefficients to zero. + Rank-deficient and underdetermined systems therefore have at most + ``rank(X)`` nonzero coefficients per frame, not a minimum-norm solution. + No normal equations or ridge regularization are used. + + Numerical rank uses the cutoff ``max(X.shape) * eps * abs(R[0, 0])``. + Pivot ties and borderline rank decisions can differ from MATLAB. + Inputs must be real float32/float64 CPU tensors without gradients; + returns ``phiᵀ`` (frames x sources), preserving the input dtype. Shared by the NMF fits here and the high-res trace solve in :mod:`giant_python.extraction.band.traces`. """ - xtx = x.T @ x - xtd = x.T @ data_motion - regularized = xtx + 1e-10 * torch.eye(xtx.shape[0]) - return torch.linalg.solve(regularized, xtd).T + x_array = x.numpy() + data_array = data_motion.numpy() + phi = np.zeros((x.shape[1], data_motion.shape[1]), dtype=x_array.dtype) + if min(x.shape) == 0 or data_motion.shape[1] == 0: + return torch.from_numpy(phi.T) + + q, r, piv = qr(x_array, mode="economic", pivoting=True) + diagonal = np.abs(np.diag(r)) + tolerance = max(x.shape) * np.finfo(x_array.dtype).eps * diagonal[0] + # Use a leading block: stop at the first numerically dependent column. + rank = int(np.count_nonzero(np.cumprod(diagonal > tolerance))) + if rank: + phi[piv[:rank], :] = solve_triangular( + r[:rank, :rank], q[:, :rank].T @ data_array + ) + return torch.from_numpy(phi.T) def fit_phi_all_motions( diff --git a/tests/test_bandsilo_nmf.py b/tests/test_bandsilo_nmf.py index 851a788..8993d72 100644 --- a/tests/test_bandsilo_nmf.py +++ b/tests/test_bandsilo_nmf.py @@ -2,10 +2,9 @@ Covers the profile/init kernels, the least-squares/SNR helpers, and the full ``fit_sources`` driver (multiplicative NMF + Adam Gaussian fit + variance -sort + SNR pruning) on a small synthetic geometry. The driver is additionally -cross-checked bit-for-bit against a verbatim copy of the reference (under a -shared torch seed) in the project's development notes; here we assert -structural behavior and determinism. +sort + SNR pruning) on a small synthetic geometry. We assert structural +behavior and determinism rather than bit-for-bit agreement with the +reference's regularized normal-equation solve. """ import unittest @@ -126,6 +125,117 @@ def test_reorder_sources(self): np.testing.assert_array_equal(xs[0][0].numpy(), [2, 0, 1]) +class TestSolvePhiMotion(unittest.TestCase): + """Unregularized, rank-revealing QR temporal solve.""" + + def test_overdetermined_multiple_frames(self): + """QR matches an independent SVD least-squares reference.""" + for dtype in (torch.float32, torch.float64): + with self.subTest(dtype=dtype): + x = torch.tensor( + [[1, 2], [3, 1], [2, 4], [0, 1]], dtype=dtype + ) + data = torch.tensor( + [[2, 1], [0, 3], [4, 2], [1, 5]], dtype=dtype + ) + phi = nmf.solve_phi_motion(x, data) + expected = np.linalg.lstsq( + x.numpy(), data.numpy(), rcond=None + )[0].T + self.assertEqual(phi.shape, (2, 2)) + self.assertEqual(phi.dtype, dtype) + self.assertEqual(phi.device, x.device) + np.testing.assert_allclose( + phi.numpy(), expected, rtol=1e-5, atol=1e-6 + ) + + def test_small_profiles_are_not_regularized(self): + """Tiny profile magnitudes must not shrink the temporal weights.""" + x = 1e-6 * torch.tensor([[1., 0.], [0., 1.], [1., 1.]]) + expected = torch.tensor([[2., 3.], [4., 5.]]) + phi = nmf.solve_phi_motion(x, x @ expected.T) + torch.testing.assert_close(phi, expected) + + def test_nearly_collinear_profiles(self): + """QR resolves columns whose float32 normal equations lose rank.""" + x = torch.tensor([[1., 1.], [1., 1.0001], [1., 0.9999]]) + expected = torch.tensor([[2., -1.], [-3., 4.]]) + phi = nmf.solve_phi_motion(x, x @ expected.T) + torch.testing.assert_close(phi, expected, rtol=5e-3, atol=5e-3) + + def test_rank_deficient_basic_solution(self): + """Dependent coefficients are zero, not shared by minimum norm.""" + for dtype in (torch.float32, torch.float64): + with self.subTest(dtype=dtype): + x = torch.tensor([[1, 2], [2, 4], [3, 6]], dtype=dtype) + # Include an inconsistent RHS to test least squares too. + data = torch.tensor([[3, 1], [6, 0], [9, 0]], dtype=dtype) + phi = nmf.solve_phi_motion(x, data) + expected = torch.tensor( + [[0, 1.5], [0, 1 / 28]], dtype=dtype + ) + torch.testing.assert_close(phi, expected) + self.assertTrue(torch.all(phi[:, 0] == 0)) + # Residual is orthogonal to the column space. + torch.testing.assert_close( + x.T @ (x @ phi.T - data), + torch.zeros((2, 2), dtype=dtype), + atol=2e-5, rtol=0, + ) + + def test_underdetermined_basic_solution(self): + """Solve only the independent pivot columns and undo permutation.""" + for dtype in (torch.float32, torch.float64): + with self.subTest(dtype=dtype): + # Distinct pivot norms avoid ambiguous tie-breaking. + x = torch.tensor([[1, 0, 2], [0, 3, 0]], dtype=dtype) + data = torch.tensor([[4, -2], [9, 6]], dtype=dtype) + phi = nmf.solve_phi_motion(x, data) + expected = torch.tensor([[0, 3, 2], [0, 2, -1]], dtype=dtype) + torch.testing.assert_close(phi, expected) + self.assertTrue(torch.all(phi[:, 0] == 0)) + torch.testing.assert_close(x @ phi.T, data) + + def test_rank_deficient_underdetermined(self): + """A wide rank-one system uses exactly one pivot coefficient.""" + x = torch.tensor([[0., 1., 2.], [0., 2., 4.]]) + data = torch.tensor([[6., -2.], [12., -4.]]) + phi = nmf.solve_phi_motion(x, data) + torch.testing.assert_close( + phi, torch.tensor([[0., 0., 3.], [0., 0., -1.]]) + ) + self.assertTrue(torch.all(phi[:, :2] == 0)) + + def test_zero_rank(self): + """A zero design matrix returns the all-zero basic solution.""" + x = torch.zeros((3, 2)) + phi = nmf.solve_phi_motion(x, torch.ones((3, 4))) + torch.testing.assert_close(phi, torch.zeros((4, 2))) + + def test_numerically_dependent_column(self): + """The dtype-relative cutoff drops a tiny trailing QR diagonal.""" + for dtype in (torch.float32, torch.float64): + with self.subTest(dtype=dtype): + x = torch.tensor( + [[1, 0], [0, torch.finfo(dtype).eps / 10], [0, 0]], + dtype=dtype, + ) + data = x @ torch.ones((2, 1), dtype=dtype) + phi = nmf.solve_phi_motion(x, data) + torch.testing.assert_close( + phi, torch.tensor([[1, 0]], dtype=dtype) + ) + + def test_empty_sources_and_frames(self): + """Empty bins and fully pruned sources preserve output dimensions.""" + for n_sources, n_frames in ((0, 3), (2, 0), (0, 0)): + with self.subTest(n_sources=n_sources, n_frames=n_frames): + x = torch.eye(4, n_sources) + data = torch.zeros((4, n_frames)) + phi = nmf.solve_phi_motion(x, data) + self.assertEqual(phi.shape, (n_frames, n_sources)) + + class TestPhiAndSnr(unittest.TestCase): """fit_phi_all_motions / build_x_support_mots / compute_source_snr.""" From 25f034081adc0e5649bf98049a80dd9510c24f6c Mon Sep 17 00:00:00 2001 From: m-xie Date: Fri, 2 Oct 2026 08:21:10 +0000 Subject: [PATCH 46/55] fix handling of nonfinite frames in trace extraction --- .../extraction/band/localization.py | 21 +++++++-- src/giant_python/extraction/band/traces.py | 3 +- tests/test_bandsilo_nmf.py | 46 +++++++++++++++++++ tests/test_bandsilo_traces.py | 33 +++++++++++++ 4 files changed, 98 insertions(+), 5 deletions(-) diff --git a/src/giant_python/extraction/band/localization.py b/src/giant_python/extraction/band/localization.py index 73083c4..b0100c2 100644 --- a/src/giant_python/extraction/band/localization.py +++ b/src/giant_python/extraction/band/localization.py @@ -282,7 +282,10 @@ def solve_phi_motion( Numerical rank uses the cutoff ``max(X.shape) * eps * abs(R[0, 0])``. Pivot ties and borderline rank decisions can differ from MATLAB. Inputs must be real float32/float64 CPU tensors without gradients; - returns ``phiᵀ`` (frames x sources), preserving the input dtype. + ``x`` must be finite. Data frames containing any NaN or infinity return + NaN for all sources; only fully finite frames enter the solve. Missing + observations are not replaced with zeros or used for partial-frame fits. + Returns ``phiᵀ`` (frames x sources), preserving the input dtype. Shared by the NMF fits here and the high-res trace solve in :mod:`giant_python.extraction.band.traces`. """ @@ -292,14 +295,24 @@ def solve_phi_motion( if min(x.shape) == 0 or data_motion.shape[1] == 0: return torch.from_numpy(phi.T) + # SciPy rejects the entire RHS batch if even one frame is nonfinite. + # Preserve missing frames without contaminating the independent fits, + # including coefficients that would otherwise be zero for deficient X. + valid_frames = np.all(np.isfinite(data_array), axis=0) + phi[:, ~valid_frames] = np.nan + q, r, piv = qr(x_array, mode="economic", pivoting=True) diagonal = np.abs(np.diag(r)) tolerance = max(x.shape) * np.finfo(x_array.dtype).eps * diagonal[0] # Use a leading block: stop at the first numerically dependent column. rank = int(np.count_nonzero(np.cumprod(diagonal > tolerance))) - if rank: - phi[piv[:rank], :] = solve_triangular( - r[:rank, :rank], q[:, :rank].T @ data_array + if rank and np.any(valid_frames): + # Avoid copying the observation matrix in the usual all-finite case. + finite_data = ( + data_array if np.all(valid_frames) else data_array[:, valid_frames] + ) + phi[np.ix_(piv[:rank], valid_frames)] = solve_triangular( + r[:rank, :rank], q[:, :rank].T @ finite_data ) return torch.from_numpy(phi.T) diff --git a/src/giant_python/extraction/band/traces.py b/src/giant_python/extraction/band/traces.py index 88b8bd8..f8ac287 100644 --- a/src/giant_python/extraction/band/traces.py +++ b/src/giant_python/extraction/band/traces.py @@ -257,7 +257,8 @@ def _solve_trial_phi_f0( ------- phi, f0 : torch.Tensor of shape (n_frames, n_sources) Per-source dF and background projection (NaN where no motion bin - applies, or where weighted averaging has no profile support). + applies, where least-squares input frames contain nonfinite samples, + or where weighted averaging has no profile support). """ n_sources = a_final.shape[1] n_frames = data.shape[1] diff --git a/tests/test_bandsilo_nmf.py b/tests/test_bandsilo_nmf.py index 8993d72..7c0deee 100644 --- a/tests/test_bandsilo_nmf.py +++ b/tests/test_bandsilo_nmf.py @@ -212,6 +212,52 @@ def test_zero_rank(self): phi = nmf.solve_phi_motion(x, torch.ones((3, 4))) torch.testing.assert_close(phi, torch.zeros((4, 2))) + def test_nonfinite_frames_do_not_abort_finite_fits(self): + """Missing observations invalidate a frame, not the whole batch.""" + for dtype in (torch.float32, torch.float64): + for design in ( + [[1, 0], [0, 1], [1, 1]], + [[1, 2], [2, 4], [3, 6]], + [[1, 0, 2], [0, 3, 0]], + [[0, 0], [0, 0], [0, 0]], + ): + with self.subTest(dtype=dtype, design=design): + x = torch.tensor(design, dtype=dtype) + data = torch.arange( + x.shape[0] * 6, dtype=dtype + ).reshape(x.shape[0], 6) + data[0, 1] = float("nan") + data[:, 2] = float("nan") + data[-1, 3] = float("inf") + data[0, 4] = -float("inf") + original = data.clone() + expected = nmf.solve_phi_motion(x, data[:, [0, 5]]) + actual = nmf.solve_phi_motion(x, data) + self.assertEqual(actual.shape, (6, x.shape[1])) + self.assertEqual(actual.dtype, dtype) + torch.testing.assert_close(actual[[0, 5]], expected) + self.assertTrue(torch.isnan(actual[1:5]).all()) + torch.testing.assert_close(data, original, equal_nan=True) + + def test_all_frames_missing(self): + """An entirely missing bin returns NaNs, including dependent sources.""" + for dtype in (torch.float32, torch.float64): + with self.subTest(dtype=dtype): + x = torch.tensor([[1, 2], [2, 4], [3, 6]], dtype=dtype) + data = torch.full((3, 4), float("nan"), dtype=dtype) + actual = nmf.solve_phi_motion(x, data) + self.assertEqual(actual.shape, (4, 2)) + self.assertEqual(actual.dtype, dtype) + self.assertTrue(torch.isnan(actual).all()) + + def test_nonfinite_profiles_still_raise(self): + """Invalid spatial models must not be silently accepted by QR.""" + for value in (float("nan"), float("inf")): + with self.subTest(value=value): + x = torch.tensor([[1., 0.], [0., value], [1., 1.]]) + with self.assertRaisesRegex(ValueError, "infs or NaNs"): + nmf.solve_phi_motion(x, torch.ones((3, 2))) + def test_numerically_dependent_column(self): """The dtype-relative cutoff drops a tiny trailing QR diagonal.""" for dtype in (torch.float32, torch.float64): diff --git a/tests/test_bandsilo_traces.py b/tests/test_bandsilo_traces.py index 2d9e2a4..43afdbd 100644 --- a/tests/test_bandsilo_traces.py +++ b/tests/test_bandsilo_traces.py @@ -192,6 +192,39 @@ def test_motion_projection_background_and_dropped_frames(self): self.assertTrue(torch.isnan(f0[-1]).all()) +class TestLeastSquaresTraceExtraction(unittest.TestCase): + """Missing high-resolution observations must not abort a trial.""" + + def test_missing_data_and_background_in_motion_bins(self): + """dF and F0 retain their own missing-frame masks across motion bins.""" + profiles = torch.tensor([[1., 2.], [3., 0.], [0., 4.]]) + h_mots = [ + torch.eye(3).to_sparse(), + torch.diag(torch.tensor([2., 1., 3.])).to_sparse(), + ] + data = np.arange(24, dtype=np.float32).reshape(3, 8) + 5 + background = np.ones_like(data) + bins = np.array([0, 0, 0, 1, 1, 1, 1, -1]) + args = ( + data, background, np.array([[0, 0], [1, 0]]), bins, + np.array([0, 1]), np.empty((0, 2)), np.empty(0), np.arange(3), + profiles, 3, 3, + ) + with patch.object(tr, "build_motion_h_matrices", return_value=h_mots): + expected_phi, expected_f0 = tr._solve_trial_phi_f0(*args) + data[0, 1] = np.nan + data[:, 2] = np.nan + background[1, 3] = np.nan + data[2, 4] = np.inf + phi, f0 = tr._solve_trial_phi_f0(*args) + + torch.testing.assert_close(phi[[0, 5, 6]], expected_phi[[0, 5, 6]]) + self.assertTrue(torch.isnan(phi[[1, 2, 3, 4, 7]]).all()) + valid_f0 = [0, 1, 2, 4, 5, 6] + torch.testing.assert_close(f0[valid_f0], expected_f0[valid_f0]) + self.assertTrue(torch.isnan(f0[[3, 7]]).all()) + + class TestBinTrialMotion(unittest.TestCase): """The configured z tolerance controls trial-frame retention.""" From 9c1e1b8df7562d57d59b33e838830f38e9dcf7be Mon Sep 17 00:00:00 2001 From: m-xie Date: Fri, 9 Oct 2026 06:00:59 +0000 Subject: [PATCH 47/55] add denoised fitting of final temporal traces --- README.md | 99 ++++- src/giant_python/cli.py | 23 +- .../extraction/band/result_assembly.py | 18 + src/giant_python/extraction/band/traces.py | 107 ++++- src/giant_python/extraction/band/transient.py | 254 +++++++++++ src/giant_python/extraction/band/types.py | 5 + src/giant_python/extraction/band/workflow.py | 20 +- src/giant_python/gui/parameters_gui.py | 27 +- src/giant_python/io/experiment_summary.py | 169 ++++++-- src/giant_python/models/experiment_summary.py | 14 +- src/giant_python/models/params.py | 44 +- tests/test_band_workflow.py | 127 ++++-- tests/test_bandsilo_pipeline.py | 177 +++++++- tests/test_bandsilo_traces.py | 234 +++++++++- tests/test_bandsilo_transient.py | 169 ++++++++ tests/test_cli.py | 61 ++- tests/test_configuration.py | 70 ++- tests/test_experiment_summary_codec.py | 405 ++++++++++++++++++ tests/test_parameter_gui.py | 54 ++- tests/test_path_summary.py | 52 ++- 20 files changed, 1964 insertions(+), 165 deletions(-) create mode 100644 src/giant_python/extraction/band/transient.py create mode 100644 tests/test_bandsilo_transient.py create mode 100644 tests/test_experiment_summary_codec.py diff --git a/README.md b/README.md index 60ec751..3b410db 100644 --- a/README.md +++ b/README.md @@ -131,27 +131,80 @@ For data not collected on SLAP2, the reference GIAnT-MATLAB pipeline sets Epochs ## Pipeline Outputs -### Optional weighted-average traces - -Set `BandSiloParams(simple_trace_extraction=True)` (or the equivalent parameter -dictionary) to extract final traces by spatial-profile weighted averaging -instead of least squares. The CLI option is `--simple-trace-extraction`; the -parameter GUI also provides a checkbox. The default is `False`. - -Source identification and profile fitting are unchanged. For each motion bin, -the fitted profile is projected into superpixel space and normalized by its -sum. The normalized intensities average the background-subtracted superpixel -activity into `dF_ls` and the background into `F0`, preserving their compatible -scales for subsequent baseline processing. Unlike least squares, this does not -de-mix overlapping sources. Unobserved frames and profiles with zero observed -support yield NaN. The output schema retains `dF_ls` in both modes, and summary -parameters record `simple_trace_extraction`. +### Least-squares and convolution-constrained traces + +Production extraction always computes and saves both standard least-squares +`sources/temporal/dF_ls` and convolution `sources/temporal/dF_denoised`, plus +the fitted `sources/temporal/events`. Each has shape +`(sources, channels, total frames)`. Only activity channel zero is processed; +other channels and missing observations are NaN. Localization/NMF/profile +fitting is unchanged. + +There are no production trace-mode switches: both outputs are mandatory. +The deprecated mode fields and CLI flags have been removed, and are no longer +saved in new summary parameters. Low-level loaded-array functions still support +weighted averages for diagnostics, not production summary extraction. + +The trace-only solver ports the standalone benchmark objective: +`0.5 * ||(data - background) - H_motion A K s||² + beta * sum(s[1:])`, +with nonnegative event amplitudes, a causal unit-peak exponential, and an +unpenalized nonnegative initial state. It solves chronologically over each +complete trial, across motion changes and missing frames; no state crosses +trial boundaries. Returned dF preserves the QR complete-frame validity mask. +F0 is still computed by QR, not by the transient model. Profiles are fixed. + +The trace kernel uses the existing `decay_tau_s` setting (default `0.15` s), +saved as `decayTau_s`, just like localization filtering. Change it with the +existing GUI Decay Tau field or the CLI `--decay-tau-s` flag. There is no +separate trace decay setting. + +Trace-only controls (also available as hyphenated CLI flags): + +| Parameter | Default | Meaning | +| --- | --- | --- | +| `trace_beta_fraction` | `0.01` | Fraction of each trial's positive adjoint scale; zero disables L1 | +| `trace_max_iter` | `500` | Maximum L-BFGS-B iterations per stage | +| `trace_tolerance` | `1e-7` | Objective/gradient tolerance | +| `trace_debias` | `False` | Refit selected events without L1 shrinkage | +| `trace_event_threshold` | `1e-6` | Absolute event threshold for debias support | + +These settings are recorded in summary parameters. A nonconverged fit raises +an error rather than silently saving an unfinished estimate. Increase the +iteration limit or inspect the low-level fit if this occurs. The standard LS +trace and its background retain the existing baseline re-estimation for +`dF_ls` and `F0`. `dF_denoised` is the fitted convolution trace itself, without +a second baseline subtraction. `events` stores nonnegative event amplitudes, +excluding the unpenalized initial state at each trial's first frame; discarded +frames are NaN. Events are not inferred across trial boundaries. + +For loaded-array use, pass `transient_options=TransientOptions(...)` and +`analyze_hz=...` to `compute_high_res_traces`. `TransientOptions` and +`fit_transient_traces` live in `giant_python.extraction.band.transient`. +The latter exposes events, initial state, in-gap model predictions, actual beta, +and convergence diagnostics, and supports an absolute `beta` or explicit causal +kernel. `TrialTraceResult` retains the original eight fields in order and adds +`d_f_denoised` and `events`. Per-trial optimizer diagnostics and latent in-gap +predictions are not added to the summary. + +This is an output-grid estimator: overlapping acquisition-line averaging is +not modeled exactly. In-gap predictions are not observations, and glutamate +events are not automatically neuronal spikes. A smoother trace alone does not +establish improved SNR. Use a fresh output location when comparing modes. ### Output schema -The extensive trees and field tables below are **reference schemas**, not a list of everything Python emits. The working BandSILo writer emits a subset of the experiment summary: source `profiles`, `coords`, `dF_ls`, `F0`, `SNR`; visualizations; `Z_depths`; global `F`; frame bookkeeping and shifts; and, when enabled, ROI `labels`, `mask`, and `F`. Disabled annotations omit the ROI group; enabled-but-empty annotations are distinct. +The BandSILo writer emits the full experiment-summary layout below: source +`profiles`, `coords`, `dF_ls`, `dF_denoised`, `events`, `F0`, `SNR`; visualizations; +column-vector `Z_depths`; global `F`; frame bookkeeping and shifts; and ROI +`labels`, `mask`, `Fsvd`, `F`. Disabled or empty annotations produce correctly +shaped zero-ROI datasets; the `annotation_enabled` group attribute distinguishes +their policies. ROI `Fsvd` is **not calculated by BandSILo** and is NaN unless +supplied explicitly through the model. It is never copied from raw `F`. -**Future/reference-only output:** `dF_denoised`, `events`, ROI `Fsvd`, and the entire per-trial summary are **not emitted**. Fields retained on model classes do not imply that the band codec reads/writes them. Arbitrary reference files are not guaranteed to round-trip losslessly through `ExperimentSummary`. +Older summaries without the new temporal datasets remain readable. Rewriting +missing optional data emits NaNs, not fabricated fits. The entire per-trial +summary remains reference-only and is not emitted. Arbitrary reference files +are not guaranteed to round-trip losslessly through `ExperimentSummary`. Trial-table writing is unavailable, and `TrialTable.to_h5` and the `AlignmentData` model have been removed. The shared HDF5 decoder and raw acquisition adapters nevertheless read the inputs needed by BandSILo. Existing summary writes replace supplied path groups but retain unrelated path groups and existing root parameters/layout flags; this is not an atomic replacement or resume mechanism. Prefer a fresh output location for a new analysis. @@ -291,8 +344,8 @@ The final step of the pipeline, source extraction (Source Identification by Acti ├ 📁 sources | ├ 📁 temporal | | ├ 📈 dF_ls (sources x channels x total frames) - | | ├ 📈 dF_denoised (sources x channels x total frames; REFERENCE ONLY, not emitted) - | | ├ 📈 events (sources x channels x total frames; REFERENCE ONLY, not emitted) + | | ├ 📈 dF_denoised (sources x channels x total frames) + | | ├ 📈 events (sources x channels x total frames) | | ├ 📈 F0 (sources x channels x total frames) | | └ 📈 SNR (sources x 1) | └ 📁 spatial @@ -301,7 +354,7 @@ The final step of the pipeline, source extraction (Source Identification by Acti ├ 📁 user_rois | ├ 🔤 labels (rois x 1) | ├ 🖼️ mask (rois x fastz x rows x cols) - | ├ 📈 Fsvd (rois x channels x total frames; REFERENCE ONLY, not emitted) + | ├ 📈 Fsvd (rois x channels x total frames; NaN when unavailable) | └ 📈 F (rois x channels x total frames) ├ 📁 visualizations | ├ 🖼️ mean_im (channels x fastz x rows x cols) @@ -494,13 +547,13 @@ Top-level fields written by **all three** motion correction backends: `numChanne | `Path{n}/global/F` | channels x total frames | numeric | Fluorescence traces over the whole field (one row per channel) | | `Path{n}/user_rois/labels` | rois x 1 | string | User-defined ROI labels | | `Path{n}/user_rois/mask` | rois x fastz x rows x cols | uint8 or bool | Stacked binary masks for each user ROI | -| `Path{n}/user_rois/Fsvd` | rois x channels x total frames | numeric | **Future/reference only; not emitted.** ROI signals after SVD / projection step | +| `Path{n}/user_rois/Fsvd` | rois x channels x total frames | numeric | ROI SVD / projection signals; NaN when unavailable (not computed by BandSILo) | | `Path{n}/user_rois/F` | rois x channels x total frames | numeric | Raw or baseline-corrected ROI fluorescence | | `Path{n}/sources/spatial/profiles` | sources x fastz x rows x cols | numeric | Spatial component / pixel weights per source, averaged across trials with footprints | | `Path{n}/sources/spatial/coords` | sources x 3 | numeric | Source centers per row: `[z_loc, y_loc, x_loc]`, **0-indexed** | | `Path{n}/sources/temporal/dF_ls` | sources x channels x total frames | numeric | Least-squares ΔF (absolute or scaled) | -| `Path{n}/sources/temporal/dF_denoised` | sources x channels x total frames | numeric | **Future/reference only; not emitted.** Denoised ΔF | -| `Path{n}/sources/temporal/events` | sources x channels x total frames | numeric | **Future/reference only; not emitted.** Deconvolved source events | +| `Path{n}/sources/temporal/dF_denoised` | sources x channels x total frames | numeric | Fitted convolution ΔF, saved alongside least squares | +| `Path{n}/sources/temporal/events` | sources x channels x total frames | numeric | Nonnegative fitted event amplitudes; excludes the trial initial state | | `Path{n}/sources/temporal/F0` | sources x channels x total frames | numeric | Baseline estimate used for normalization | | `Path{n}/sources/temporal/SNR` | sources x 1 | numeric | (optional) Signal-to-noise ratio metric; only written when extraction emits per-source SNR | diff --git a/src/giant_python/cli.py b/src/giant_python/cli.py index 014e87e..a55d08d 100644 --- a/src/giant_python/cli.py +++ b/src/giant_python/cli.py @@ -26,10 +26,20 @@ def _add_silo_options(parser: argparse.ArgumentParser) -> None: """Attach the shared source-extraction options to a subparser.""" + for name, kind, help_text in ( + ("decay_tau_s", float, "Shared localization and trace decay time in seconds."), + ("trace_beta_fraction", float, "Trial-relative nonnegative event penalty."), + ("trace_max_iter", int, "Maximum iterations per transient fit stage."), + ("trace_tolerance", float, "Transient optimizer tolerance."), + ("trace_event_threshold", float, "Absolute event threshold for debias support."), + ): + parser.add_argument( + "--" + name.replace("_", "-"), type=kind, + default=getattr(BandSiloParams, name), help=help_text, + ) parser.add_argument( - "--simple-trace-extraction", - action="store_true", - help="Use profile-weighted averages instead of least squares for final traces.", + "--trace-debias", action="store_true", + help="Refit selected transient events without the L1 penalty.", ) parser.add_argument( "--z-tol", @@ -121,7 +131,12 @@ def _options_from_args( return ( BandSiloParams( z_tol=args.z_tol, - simple_trace_extraction=args.simple_trace_extraction, + decay_tau_s=args.decay_tau_s, + trace_beta_fraction=args.trace_beta_fraction, + trace_max_iter=args.trace_max_iter, + trace_tolerance=args.trace_tolerance, + trace_debias=args.trace_debias, + trace_event_threshold=args.trace_event_threshold, ), ExecutionOptions( max_workers=args.max_workers, diff --git a/src/giant_python/extraction/band/result_assembly.py b/src/giant_python/extraction/band/result_assembly.py index 09b46d7..c4388e4 100644 --- a/src/giant_python/extraction/band/result_assembly.py +++ b/src/giant_python/extraction/band/result_assembly.py @@ -91,6 +91,20 @@ def assemble_path_summary(pr: PathResult, path_index: int = 0) -> PathSummary: ) coords = pr.source_params[:, :3].numpy() df_channels = _with_nan_channel(d_f, pr.num_channels) + # Keep the fitted convolution trace itself, not a second baseline-subtracted + # version: otherwise it would no longer correspond to the fitted events. + # Legacy manually constructed records have no transient outputs. + denoised = np.concatenate([ + r.d_f_denoised if r.d_f_denoised is not None + else np.full_like(r.d_f, np.nan) + for r in results + ], axis=0) + events = np.concatenate([ + r.events if r.events is not None else np.full_like(r.d_f, np.nan) + for r in results + ], axis=0) + denoised_channels = _with_nan_channel(denoised, pr.num_channels) + event_channels = _with_nan_channel(events, pr.num_channels) f0_channels = _with_nan_channel(f0, pr.num_channels) snr = pr.source_snr.reshape(-1, 1) sources = [ @@ -98,6 +112,8 @@ def assemble_path_summary(pr: PathResult, path_index: int = 0) -> PathSummary: profile=profiles[j], coords=coords[j], df_ls=df_channels[j], + df_denoised=denoised_channels[j], + events=event_channels[j], f0=f0_channels[j], snr=snr[j, 0], ) @@ -172,6 +188,8 @@ def assemble_path_summary(pr: PathResult, path_index: int = 0) -> PathSummary: "profiles": profiles, "coords": coords, "dF_ls": df_channels, + "dF_denoised": denoised_channels, + "events": event_channels, "F0": f0_channels, "SNR": snr, } diff --git a/src/giant_python/extraction/band/traces.py b/src/giant_python/extraction/band/traces.py index f8ac287..46e5e1f 100644 --- a/src/giant_python/extraction/band/traces.py +++ b/src/giant_python/extraction/band/traces.py @@ -7,14 +7,17 @@ low-res motion bins used during NMF), and — per motion bin — projects the fixed spatial profiles ``A_final`` into superpixel space and extracts per-source temporal traces (``phi``, stored as ``dF_ls``) and background (``F0``) via -least squares or normalized profile-weighted averages. It also computes the +least squares or normalized profile-weighted averages. Transient fitting +adds separate denoised traces and events, jointly across the trial with fixed +spatial profiles and QR F0. The production workflow always calculates both +least-squares and transient traces. It also computes the global and per-user-ROI fluorescence. The pure compute (:func:`compute_high_res_traces`) is factored out of the IO wrapper (:func:`get_high_res_traces`, which reads the SLAP2 file lazily via :mod:`giant_python.extraction.band.trial_data`) so the numerics are testable without ``slap2_utils``. The workflow owns per-trial scheduling. Results use -``TrialTraceResult``, preserving the historical eight-tuple ordering. +``TrialTraceResult``, with denoised traces and events after the original fields. """ from __future__ import annotations @@ -28,6 +31,7 @@ from .geometry import build_selected_pixel_mask, ref_pixs_to_drc from .localization import solve_phi_motion from .operators import build_motion_h_matrices +from .transient import TransientOptions, fit_transient_traces from .trial_data import read_band_trial_data from .types import TrialTraceResult @@ -49,7 +53,7 @@ def _empty_trace_result( Returns ------- tuple - Same 8-element shape as :func:`compute_high_res_traces`, with zero + Same record as :func:`compute_high_res_traces`, with zero frames. """ return TrialTraceResult( @@ -65,6 +69,8 @@ def _empty_trace_result( np.full((0,), 0, dtype=np.int16), ), np.full((0, n_soma, num_channels), np.nan, dtype=np.float32), + np.full((0, n_sources), np.nan, dtype=np.float32), + np.full((0, n_sources), np.nan, dtype=np.float32), ) @@ -219,7 +225,11 @@ def _solve_trial_phi_f0( num_super_pixels: int, dmd_pixels_per_row: int, simple_trace_extraction: bool = False, -) -> Tuple[torch.Tensor, torch.Tensor]: + *, + transient_options: Optional[TransientOptions] = None, + analyze_hz: Optional[float] = None, + return_transient: bool = False, +) -> tuple: """Extract per-source traces and background projection. For each kept motion bin, shifts ``H`` by the bin's motion, projects @@ -252,6 +262,15 @@ def _solve_trial_phi_f0( simple_trace_extraction : bool, optional Use normalized profile-intensity weighted averages instead of least squares for both dF and F0 (default False). + transient_options : TransientOptions or None + Add benchmark-style constrained dF, keeping separate QR dF and F0. Mutually + exclusive with simple extraction. A failed fit raises RuntimeError. + analyze_hz : float or None + Nominal grid rate, required for transient fitting; line indices are + not seconds. Missing frames must retain their positions in the grid. + return_transient : bool + Also return denoised dF and events after phi and f0. Missing-frame + events are NaN; the initial state is not counted as a new event. Returns ------- @@ -260,10 +279,20 @@ def _solve_trial_phi_f0( applies, where least-squares input frames contain nonfinite samples, or where weighted averaging has no profile support). """ + if transient_options is not None: + if simple_trace_extraction: + raise ValueError("Simple and convolution trace extraction are mutually exclusive") + if not isinstance(transient_options, TransientOptions): + raise TypeError("transient_options must be TransientOptions or None") + from ...models.params import _positive + + _positive("analyze_hz", analyze_hz) n_sources = a_final.shape[1] n_frames = data.shape[1] phi = torch.full((n_frames, n_sources), float("nan"), dtype=torch.float32) f0 = torch.full((n_frames, n_sources), float("nan"), dtype=torch.float32) + denoised = torch.full_like(phi, float("nan")) + events = torch.full_like(phi, float("nan")) residual = data - background # One shifted superpixel<-pixel H per trial motion bin (same builder the @@ -280,19 +309,43 @@ def _solve_trial_phi_f0( extract = ( _weighted_trace_average if simple_trace_extraction else solve_phi_motion ) + designs = {} for motion_idx in mot_inds_to_keep: motion_frames = np.flatnonzero(mot_inds == motion_idx) x = torch.sparse.mm(h_mots[motion_idx], a_final[sel_pix_idxs, :]) - # phi from the background-subtracted residual; F0 from the background, - # both using the same extraction method and spatial profiles. + # F0 remains QR for transient fitting (never impose events on F0). phi[motion_frames, :] = extract( x, torch.from_numpy(residual[:, motion_frames].astype(np.float32)) ) + if transient_options is not None: + designs[int(motion_idx)] = x.detach().cpu().numpy() f0[motion_frames, :] = extract( x, torch.from_numpy(background[:, motion_frames].astype(np.float32)), ) + if transient_options is not None and n_sources: + residual32 = residual.astype(np.float32) + valid = np.isin(mot_inds, mot_inds_to_keep) + valid &= np.all(np.isfinite(residual32), axis=0) + # Preserve the empty/all-invalid behavior of the existing extractor. + if np.any(valid): + result = fit_transient_traces( + residual32, designs, mot_inds, valid, analyze_hz, + options=transient_options, + ) + if not result.metadata["converged"]: + raise RuntimeError( + "Convolution trace extraction did not converge: " + f"{result.metadata['message']}. Increase trace_max_iter " + "or inspect the fit with fit_transient_traces()." + ) + denoised = torch.from_numpy(result.d_f.astype(np.float32)) + events = torch.from_numpy(result.events.astype(np.float32)) + events[~valid] = float("nan") + + if return_transient: + return phi, f0, denoised, events return phi, f0 @@ -392,6 +445,9 @@ def compute_high_res_traces( soma_sps: List[np.ndarray], z_tol: float = 1.5, simple_trace_extraction: bool = False, + *, + transient_options: Optional[TransientOptions] = None, + analyze_hz: Optional[float] = None, ) -> TrialTraceResult: """Compute one trial's high-res source traces from loaded arrays. @@ -433,14 +489,29 @@ def compute_high_res_traces( simple_trace_extraction : bool, optional Average superpixels weighted by normalized source profiles instead of least-squares fitting final dF and F0 traces (default False). + transient_options : TransientOptions or None + Add constrained dF and events, retaining QR dF/F0 and auxiliary data. + Fits nonnegative sparse events plus an unpenalized initial state. + Public dF retains missing-frame NaNs; nonconvergence raises RuntimeError. + analyze_hz : float or None + Required nominal rate for transient fitting on the uniform full grid. + Acquisition averaging is approximated, not explicitly modeled. Returns ------- - tuple - ``(phi, F0, frames, sel_pix_idxs, global_f, (motion_r, motion_c, - motion_z), (online_y, online_x, online_z), F_soma)``. ``phi`` and - ``F0`` are ``(n_frames, n_sources)`` ndarrays. + TrialTraceResult + Named source traces and auxiliary data. Source trace arrays are + ``(n_frames, n_sources)``; denoised traces/events are separate from d_f. """ + if transient_options is not None: + frame_grid = np.asarray(frames) + if frame_grid.ndim != 1 or not np.all(np.isfinite(frame_grid)): + raise ValueError("frames must be a finite chronological vector") + # Reader ceil-rounds line positions, so nominally uniform samples + # can have alternating line spacings. Use the explicit nominal rate. + steps = np.diff(frame_grid) + if np.any(steps <= 0): + raise ValueError("Convolution traces require chronological frames; retain gaps") ( motion_r, motion_c, @@ -480,7 +551,7 @@ def compute_high_res_traces( psf2d, ) - phi, f0 = _solve_trial_phi_f0( + phi, f0, denoised, events = _solve_trial_phi_f0( data, background, unique_motion, @@ -493,6 +564,9 @@ def compute_high_res_traces( num_super_pixels, dmd_pixels_per_row, simple_trace_extraction=simple_trace_extraction, + transient_options=transient_options, + analyze_hz=analyze_hz, + return_transient=True, ) global_f = _compute_global_f(data, data2, frames_to_keep, num_channels) @@ -509,6 +583,8 @@ def compute_high_res_traces( (motion_r, motion_c, motion_z), (online_y, online_x, online_z), f_soma, + denoised.numpy(), + events.numpy(), ) @@ -573,6 +649,8 @@ def get_high_res_traces( soma_sps: List[np.ndarray], z_tol: float = 1.5, simple_trace_extraction: bool = False, + *, + transient_options: Optional[TransientOptions] = None, ) -> TrialTraceResult: """Read and extract one trial's high-res source traces. @@ -616,11 +694,14 @@ def get_high_res_traces( simple_trace_extraction : bool, optional Average superpixels weighted by normalized source profiles instead of least-squares fitting final dF and F0 traces (default False). + transient_options : TransientOptions or None + Add constrained dF/events alongside QR dF/F0 and unchanged profiles. + The reader's samp_freq supplies the nominal convolution rate. Returns ------- tuple - The 8-element result of :func:`compute_high_res_traces` (empty for + The named result of :func:`compute_high_res_traces` (empty for skipped trials). """ trial_ix, keep_trial, background_ds = trial_info @@ -664,4 +745,6 @@ def get_high_res_traces( soma_sps, z_tol=z_tol, simple_trace_extraction=simple_trace_extraction, + transient_options=transient_options, + analyze_hz=samp_freq, ) diff --git a/src/giant_python/extraction/band/transient.py b/src/giant_python/extraction/band/transient.py new file mode 100644 index 0000000..4d41924 --- /dev/null +++ b/src/giant_python/extraction/band/transient.py @@ -0,0 +1,254 @@ +"""Fixed-profile, output-grid sparse transient fitting. + +Port of the standalone trace benchmark, not a localization/NMF solver. +Fits 0.5 * ||r - X K s||^2 + beta * sum(s[1:]), s >= 0, using +float64 sufficient statistics and bounded L-BFGS-B. The first coefficient +is an unpenalized initial state. Trials must be solved independently. +Reader averaging is not modeled exactly; events are not spike times. +""" + +from dataclasses import dataclass +import warnings + +import numpy as np +from scipy.optimize import minimize +from scipy.signal import fftconvolve, lfilter + +from ...models.params import _nonnegative, _positive + + +@dataclass(frozen=True) +class TransientOptions: + """Settings for trace-only fitting with fixed spatial profiles. + + The workflow supplies tau_s from the shared BandSiloParams.decay_tau_s. + ``beta`` overrides the trial-relative ``beta_fraction`` when supplied. + ``debias`` optionally refits selected events without the L1 penalty. + ``event_threshold`` is in absolute coefficient units, not scaled units. + """ + + tau_s: float = 0.15 + beta_fraction: float = 0.01 + beta: float | None = None + max_iter: int = 500 + tolerance: float = 1e-7 + debias: bool = False + event_threshold: float = 1e-6 + + def __post_init__(self): + """Reject invalid solver settings before fitting or reading data.""" + _positive("tau_s", self.tau_s) + _positive("max_iter", self.max_iter, integer=True) + _positive("tolerance", self.tolerance) + _nonnegative("beta_fraction", self.beta_fraction) + _nonnegative("event_threshold", self.event_threshold) + if self.beta is not None: + _nonnegative("beta", self.beta) + if not isinstance(self.debias, bool): + raise ValueError("debias must be a bool") + + +@dataclass +class TransientResult: + """Trace, latent predictions/events, and explicit convergence diagnostics.""" + + d_f: np.ndarray + metadata: dict + events: np.ndarray + initial_state: np.ndarray + model_trace: np.ndarray + + +class TransientOperator: + """Causal convolution and exact adjoint without a dense temporal matrix.""" + + def __init__(self, analyze_hz, tau_s, kernel=None): + """Use a unit-peak exponential or an explicit nonnegative causal kernel.""" + _positive("analyze_hz", analyze_hz) + _positive("tau_s", tau_s) + self.gamma = float(np.exp(-1 / (analyze_hz * tau_s))) + self.kernel = None if kernel is None else np.asarray(kernel, dtype=float) + if self.kernel is not None and ( + self.kernel.ndim != 1 or not self.kernel.size + or not np.all(np.isfinite(self.kernel)) + or np.any(self.kernel < 0) or self.kernel[0] <= 0 + ): + raise ValueError("kernel must be finite, nonnegative, with kernel[0]>0") + + def forward(self, events): + """Convolve all sources in chronological order, including gaps.""" + if self.kernel is None: + return lfilter([1.0], [1.0, -self.gamma], events, axis=0) + return fftconvolve( + events, self.kernel[:, None], mode="full", axes=0 + )[:len(events)] + + def adjoint(self, traces): + """Apply the exact transpose, including finite-trial boundaries.""" + return self.forward(traces[::-1])[::-1] + + +def _temporal_quadratic(normal, groups, trace): + """Half squared error minus its data constant, and the trace gradient.""" + gradient = np.zeros_like(trace) + value = 0.0 + for motion, (gram, rhs) in normal.items(): + ix = groups[motion] + fit = trace[ix] + gt = fit @ gram + value += np.sum(fit * (0.5 * gt - rhs.T)) + gradient[ix] = gt - rhs.T + return float(value), gradient + + +def fit_transient_traces( + residual, designs, motion_indices, valid, analyze_hz, + *, options=None, kernel=None, +): + """Fit one trial using fixed motion-specific (superpixel, source) designs. + + Residual is (superpixels, frames). The complete nominal output grid must + be retained, in chronological order: never compress out gaps or group + disjoint motion frames before convolution. Nonfinite observations reject + the entire frame, as in QR. ``valid`` also excludes unmatched motion/z. + Public dF is NaN there, while model_trace contains latent gap predictions. + + Low-level nonconvergence returns diagnostics and warns. Production callers + must check ``metadata['converged']`` before saving a result. + """ + options = TransientOptions() if options is None else options + if not isinstance(options, TransientOptions): + raise TypeError("options must be TransientOptions or None") + op = TransientOperator(analyze_hz, options.tau_s, kernel) + residual = np.asarray(residual) + motion_indices = np.asarray(motion_indices) + valid = np.asarray(valid, dtype=bool).copy() + if residual.ndim != 2: + raise ValueError("residual must have shape (superpixels, frames)") + n_sp, n_frames = residual.shape + if valid.shape != (n_frames,) or motion_indices.shape != (n_frames,): + raise ValueError("motion_indices and valid must have one entry per frame") + if not designs: + raise ValueError("Transient fitting requires projected profiles") + arrays = {m: np.asarray(x) for m, x in designs.items()} + first = next(iter(arrays.values())) + if first.ndim != 2: + raise ValueError("designs must have shape (superpixels, sources)") + n_sources = first.shape[1] + for x in arrays.values(): + if x.shape != (n_sp, n_sources) or not np.all(np.isfinite(x)): + raise ValueError("Projected profiles must be finite with matching shapes") + valid &= np.all(np.isfinite(residual), axis=0) + valid &= np.isin(motion_indices, list(arrays)) + if not np.any(valid) or n_sources == 0: + raise ValueError("Transient fitting requires observed frames and sources") + groups, normal = {}, {} + for motion, x in arrays.items(): + ix = np.flatnonzero(valid & (motion_indices == motion)) + if ix.size: + xd = x.astype(np.float64) + groups[motion] = ix + normal[motion] = ( + xd.T @ xd, xd.T @ residual[:, ix].astype(np.float64) + ) + + shape = (n_frames, n_sources) + rhs_trace = np.zeros(shape) + for motion, (_, rhs) in normal.items(): + rhs_trace[groups[motion]] = rhs.T + event_rhs = op.adjoint(rhs_trace) + beta_scale = float(max(0.0, np.max(event_rhs[1:], initial=0.0))) + penalty = float(options.beta if options.beta is not None + else options.beta_fraction * beta_scale) + diagonals = np.concatenate([np.diag(g) for g, _ in normal.values()]) + positive = diagonals[diagonals > 0] + if not positive.size: + raise ValueError("No observable profile support") + gram_scale = float(np.max(positive)) + amplitude_scale = max(float(np.max(np.abs(rhs_trace))) / gram_scale, 1e-12) + objective_scale = gram_scale * amplitude_scale ** 2 + + def objective(flat, event_penalty=penalty): + """Evaluate the scaled objective and its analytic event gradient.""" + coefficients = flat.reshape(shape) * amplitude_scale + trace = op.forward(coefficients) + value, trace_gradient = _temporal_quadratic(normal, groups, trace) + gradient = op.adjoint(trace_gradient).copy() + value += event_penalty * coefficients[1:].sum() + gradient[1:] += event_penalty + return ( + value / objective_scale, + (gradient * amplitude_scale / objective_scale).ravel(), + ) + + optimizer_options = { + "maxiter": options.max_iter, "ftol": options.tolerance, + "gtol": options.tolerance, "maxls": 40, "maxcor": 10, + } + result = minimize( + objective, np.zeros(np.prod(shape)), method="L-BFGS-B", jac=True, + bounds=[(0.0, None)] * int(np.prod(shape)), options=optimizer_options, + ) + selection = { + "converged": bool(result.success), "iterations": int(result.nit), + "message": str(result.message), + "objective_without_data_constant": float(result.fun * objective_scale), + } + debias_info = { + "requested": options.debias, "performed": False, + "event_threshold": float(options.event_threshold), + } + if options.debias and result.success: + active = result.x.reshape(shape) * amplitude_scale > options.event_threshold + active[0] = True + free = np.flatnonzero(active.ravel()) + debias_info["retained_events"] = int(active[1:].sum()) + + def restricted_objective(reduced): + """Refit selected events only; the initial state remains free.""" + full = np.zeros(np.prod(shape)) + full[free] = reduced + value, gradient = objective(full, event_penalty=0.0) + return value, gradient[free] + + refit = minimize( + restricted_objective, result.x[free], method="L-BFGS-B", jac=True, + bounds=[(0.0, None)] * len(free), options=optimizer_options, + ) + full = np.zeros(np.prod(shape)) + full[free] = refit.x + refit.x = full + result = refit + debias_info.update( + performed=True, converged=bool(refit.success), + iterations=int(refit.nit), message=str(refit.message), + ) + elif options.debias: + debias_info["skip_reason"] = "Support-selection fit did not converge" + coefficients = result.x.reshape(shape) * amplitude_scale + model_trace = op.forward(coefficients) + phi = model_trace.copy() + phi[~valid] = np.nan + events = coefficients.copy() + initial = events[0].copy() + events[0] = 0 + converged = bool( + result.success and np.isfinite(result.fun) + and np.all(np.isfinite(model_trace)) + ) + if not converged: + warnings.warn( + f"Transient solver did not converge: {result.message}", + RuntimeWarning, stacklevel=2, + ) + return TransientResult(phi, { + "method": "transient", "tau_s": options.tau_s, "beta": penalty, + "beta_fraction": options.beta_fraction if options.beta is None else None, + "beta_scale": beta_scale, "converged": converged, + "iterations": int(result.nit), "message": str(result.message), + "kernel": "exponential" if kernel is None else np.asarray(kernel).tolist(), + "sampling_model": "nominal output grid; acquisition averaging not modeled exactly", + "selection": selection, "debias": debias_info, + "final_event_penalty": 0.0 if debias_info["performed"] else penalty, + "objective_without_data_constant": float(result.fun * objective_scale), + }, events, initial, model_trace) \ No newline at end of file diff --git a/src/giant_python/extraction/band/types.py b/src/giant_python/extraction/band/types.py index 86a6e26..fce41d2 100644 --- a/src/giant_python/extraction/band/types.py +++ b/src/giant_python/extraction/band/types.py @@ -14,6 +14,9 @@ class TrialTraceResult(NamedTuple): Reference offsets and online shifts are (row, column, z) arrays, each with one value per frame. Only output assembly negates reference offsets into saved MATLAB displacement. Skipped trials have zero frames. + ``d_f`` is the LS result in production; ``d_f_denoised`` and ``events`` + contain separate convolution outputs. Optional defaults permit legacy + eight-argument construction, but the record now has ten fields. """ d_f: np.ndarray @@ -24,6 +27,8 @@ class TrialTraceResult(NamedTuple): reference_offsets: tuple[np.ndarray, np.ndarray, np.ndarray] online_shifts: tuple[np.ndarray, np.ndarray, np.ndarray] user_roi_f: np.ndarray + d_f_denoised: np.ndarray | None = None + events: np.ndarray | None = None @dataclass(frozen=True) diff --git a/src/giant_python/extraction/band/workflow.py b/src/giant_python/extraction/band/workflow.py index 4deab28..4fda820 100644 --- a/src/giant_python/extraction/band/workflow.py +++ b/src/giant_python/extraction/band/workflow.py @@ -160,7 +160,11 @@ def _params_dict( "vif": params.vif, "peakth": params.peakth, "peak_buffer": params.peak_buffer, - "simple_trace_extraction": params.simple_trace_extraction, + "trace_beta_fraction": params.trace_beta_fraction, + "trace_max_iter": params.trace_max_iter, + "trace_tolerance": params.trace_tolerance, + "trace_debias": params.trace_debias, + "trace_event_threshold": params.trace_event_threshold, "draw_user_rois": annotations.enabled, } @@ -322,6 +326,18 @@ def _read(trial_ix): execution=execution, ) + # Trace-only options: never passed to localization or its QR solver. + from .transient import TransientOptions + + transient_options = TransientOptions( + tau_s=params.decay_tau_s, + beta_fraction=params.trace_beta_fraction, + max_iter=params.trace_max_iter, + tolerance=params.trace_tolerance, + debias=params.trace_debias, + event_threshold=params.trace_event_threshold, + ) + def _traces(trial_ix): """Extract high-res traces using this trial's background slice.""" return get_high_res_traces( @@ -350,7 +366,7 @@ def _traces(trial_ix): num_channels, soma_sps, z_tol=params.z_tol, - simple_trace_extraction=params.simple_trace_extraction, + transient_options=transient_options, ) trace_results = map_trials( diff --git a/src/giant_python/gui/parameters_gui.py b/src/giant_python/gui/parameters_gui.py index d5c9cfc..47d40eb 100644 --- a/src/giant_python/gui/parameters_gui.py +++ b/src/giant_python/gui/parameters_gui.py @@ -65,11 +65,26 @@ def run_parameter_gui( sparse_var = tk.StringVar(value=str(float(np.log(base.sparse_fac)))) ttk.Entry(frame, textvariable=sparse_var, width=15).grid(row=row, column=1) row += 1 - ttk.Label(frame, text="Simple Trace Extraction?").grid( + ttk.Label(frame, text="LS and convolution traces are always saved.").grid( + row=row, column=0, columnspan=2, sticky=tk.W + ) + row += 1 + for attr, label in ( + ("trace_beta_fraction", "Trace Event Penalty Fraction:"), + ("trace_max_iter", "Trace Maximum Iterations:"), + ("trace_tolerance", "Trace Optimizer Tolerance:"), + ("trace_event_threshold", "Trace Debias Event Threshold:"), + ): + ttk.Label(frame, text=label).grid(row=row, column=0, sticky=tk.W) + var = tk.StringVar(value=str(values[attr])) + ttk.Entry(frame, textvariable=var, width=15).grid(row=row, column=1) + entries[attr] = var + row += 1 + ttk.Label(frame, text="Debias Convolution Traces?").grid( row=row, column=0, sticky=tk.W ) - simple_trace_var = tk.BooleanVar(value=base.simple_trace_extraction) - ttk.Checkbutton(frame, variable=simple_trace_var).grid(row=row, column=1) + debias_var = tk.BooleanVar(value=base.trace_debias) + ttk.Checkbutton(frame, variable=debias_var).grid(row=row, column=1) row += 1 ttk.Label(frame, text="Draw User ROIs?").grid( row=row, column=0, sticky=tk.W @@ -96,7 +111,11 @@ def on_ok(): "peak_buffer": int(entries["peak_buffer"].get()), "z_tol": float(entries["z_tol"].get()), "sparse_fac": float(np.exp(float(sparse_var.get()))), - "simple_trace_extraction": bool(simple_trace_var.get()), + "trace_beta_fraction": float(entries["trace_beta_fraction"].get()), + "trace_max_iter": int(entries["trace_max_iter"].get()), + "trace_tolerance": float(entries["trace_tolerance"].get()), + "trace_event_threshold": float(entries["trace_event_threshold"].get()), + "trace_debias": bool(debias_var.get()), } result["options"] = ( replace(base, **updates), diff --git a/src/giant_python/io/experiment_summary.py b/src/giant_python/io/experiment_summary.py index 6523b60..cb3a924 100644 --- a/src/giant_python/io/experiment_summary.py +++ b/src/giant_python/io/experiment_summary.py @@ -26,7 +26,8 @@ ) from .hdf5 import load_struct_h5, write_dict_to_h5group -_SOURCE_KEYS = ("profiles", "coords", "dF_ls", "F0", "SNR") +_TEMPORAL_KEYS = ("dF_ls", "dF_denoised", "events", "F0", "SNR") +_SOURCE_KEYS = ("profiles", "coords", *_TEMPORAL_KEYS) def _decode_path(assembled: dict, path_index: int) -> PathSummary: @@ -40,6 +41,11 @@ def _decode_path(assembled: dict, path_index: int) -> PathSummary: profile=assembled["profiles"][j], coords=assembled["coords"][j], df_ls=assembled["dF_ls"][j], + df_denoised=( + assembled["dF_denoised"][j] + if "dF_denoised" in assembled else None + ), + events=assembled["events"][j] if "events" in assembled else None, f0=assembled["F0"][j], snr=assembled["SNR"][j, 0], ) @@ -54,6 +60,11 @@ def _decode_path(assembled: dict, path_index: int) -> PathSummary: label=str(label), mask=rois["mask"][j] if j < rois["mask"].shape[0] else None, f=rois["F"][j] if j < rois["F"].shape[0] else None, + fsvd=( + rois["Fsvd"][j] + if "Fsvd" in rois and j < rois["Fsvd"].shape[0] + else None + ), ) for j, label in enumerate(labels) ] @@ -71,12 +82,16 @@ def _decode_path(assembled: dict, path_index: int) -> PathSummary: ), global_f=assembled["global_f"], z_depths=assembled["Z_depths"], - annotation_enabled=rois is not None, + annotation_enabled=assembled.get( + "annotation_enabled", rois is not None + ), _empty_source_arrays=( - {} if sources else {key: assembled[key] for key in _SOURCE_KEYS} + {} if sources else { + key: assembled[key] for key in _SOURCE_KEYS if key in assembled + } ), _empty_roi_arrays=( - {key: rois[key] for key in ("mask", "F")} + {key: rois[key] for key in ("mask", "F", "Fsvd") if key in rois} if rois is not None and not user_rois else {} ), @@ -95,12 +110,54 @@ def _stack_required_arrays( return np.stack(required) +def _optional_array(array, reference: np.ndarray, field: str) -> np.ndarray: + """Validate a supplied trace, or mark unavailable samples with NaNs.""" + if array is None: + dtype = ( + reference.dtype + if np.issubdtype(reference.dtype, np.inexact) + else np.dtype(float) + ) + return np.full(reference.shape, np.nan, dtype=dtype) + array = np.asarray(array) + if array.shape != reference.shape: + raise ValueError( + f"{field} must have shape {reference.shape}; got {array.shape}" + ) + return array + + +def _stack_optional_arrays( + arrays: Iterable[np.ndarray | None], reference: np.ndarray, field: str +) -> np.ndarray: + """Check each optional entry against its corresponding required trace.""" + return np.stack([ + _optional_array(array, reference[index], f"{field}[{index}]") + for index, array in enumerate(arrays) + ]) + + +def _empty_geometry( + path: PathSummary, field: str +) -> tuple[tuple[int, ...], np.ndarray]: + """Infer empty tensor dimensions from the saved images and global trace.""" + mean = np.asarray(path.visualizations.mean_im) + global_f = np.asarray(path.global_f) + if mean.ndim != 4 or global_f.ndim != 2: + raise ValueError( + f"Empty {field} require shape templates or " + "mean_im and global_f geometry" + ) + return mean.shape[1:], global_f + + def _encode_path(path: PathSummary) -> dict: """Expose band-schema arrays for persistence, without scientific changes. Populated lists are stacked so edits to public Source/UserRoi entries are reflected. Empty results retain their original tensor dimensions/dtypes. - Missing required Source/UserRoi arrays raise ValueError before persistence. + Missing optional traces are NaN, never inferred from other signals. + Required fields and optional shapes are checked before path replacement. """ out = { "Z_depths": path.z_depths, @@ -125,24 +182,36 @@ def _encode_path(path: PathSummary) -> dict: "F0": _stack_required_arrays( (s.f0 for s in path.sources), "Source.f0" ), - "SNR": np.asarray([s.snr for s in path.sources]).reshape( - -1, 1 - ), + "SNR": np.asarray([ + np.nan if s.snr is None else s.snr for s in path.sources + ]).reshape(-1, 1), } ) + for key, field in (("dF_denoised", "df_denoised"), ("events", "events")): + out[key] = _stack_optional_arrays( + (getattr(s, field) for s in path.sources), + out["dF_ls"], + f"Source.{field}", + ) elif path._empty_source_arrays: out.update(path._empty_source_arrays) else: + spatial_shape, global_f = _empty_geometry(path, "sources") + out.update({ + "profiles": np.empty((0, *spatial_shape), dtype=np.float32), + "coords": np.empty((0, 3), dtype=np.int32), + "dF_ls": np.empty((0, *global_f.shape), dtype=global_f.dtype), + "F0": np.empty((0, *global_f.shape), dtype=global_f.dtype), + "SNR": np.empty((0, 1), dtype=np.float32), + }) + if not path.sources: + for key in ("dF_denoised", "events"): + out[key] = _optional_array(out.get(key), out["dF_ls"], key) + if not path.annotation_enabled and path.user_rois: raise ValueError( - "Empty source lists require band array shape templates" + "Set annotation_enabled=True to serialize user ROIs" ) - if not path.annotation_enabled: - if path.user_rois: - raise ValueError( - "Set annotation_enabled=True to serialize user ROIs" - ) - out["user_rois"] = None - elif path.user_rois: + if path.user_rois: out["user_rois"] = { "mask": _stack_required_arrays( (roi.mask for roi in path.user_rois), "UserRoi.mask" @@ -152,22 +221,31 @@ def _encode_path(path: PathSummary) -> dict: (roi.f for roi in path.user_rois), "UserRoi.f" ), } + out["user_rois"]["Fsvd"] = _stack_optional_arrays( + (roi.fsvd for roi in path.user_rois), + out["user_rois"]["F"], + "UserRoi.fsvd", + ) else: if path._empty_roi_arrays: empty = path._empty_roi_arrays else: - # Enabled-but-empty annotations constructed directly by callers. - mean = np.asarray(path.visualizations.mean_im) - global_f = np.asarray(path.global_f) - if mean.ndim != 4 or global_f.ndim != 2: - raise ValueError( - "Empty ROIs require mean_im and global_f geometry" - ) + # Both disabled and enabled-but-empty annotations have full schema. + spatial_shape, global_f = _empty_geometry(path, "ROIs") empty = { - "mask": np.zeros((0, *mean.shape[1:]), dtype=bool), + "mask": np.zeros((0, *spatial_shape), dtype=bool), "F": np.empty((0, *global_f.shape), dtype=global_f.dtype), } - out["user_rois"] = {**empty, "labels": []} + out["user_rois"] = { + **empty, + "labels": [], + "Fsvd": _optional_array( + empty.get("Fsvd"), empty["F"], "UserRoi.Fsvd" + ), + } + if path.z_depths is None: + raise ValueError("Z_depths is required; got None") + out["Z_depths"] = np.asarray(path.z_depths).reshape(-1, 1) return out @@ -184,7 +262,7 @@ def _write_path_group(f: h5py.File, path: PathSummary) -> None: for key in ("profiles", "coords"): spatial.create_dataset(key, data=assembled[key]) temporal = sources.create_group("temporal") - for key in ("dF_ls", "F0", "SNR"): + for key in _TEMPORAL_KEYS: temporal.create_dataset(key, data=assembled[key]) frame_group = group.create_group("frame_info") for key in FRAME_INFO_KEYS: @@ -195,23 +273,25 @@ def _write_path_group(f: h5py.File, path: PathSummary) -> None: vis = group.create_group("visualizations") for key in ("act_im", "mean_im", "act_im_peaks"): vis.create_dataset(key, data=assembled[key]) - if assembled["user_rois"] is not None: - rois = assembled["user_rois"] - roi_group = group.create_group("user_rois") - roi_group.create_dataset("mask", data=rois["mask"]) - roi_group.create_dataset( - "labels", - data=np.asarray(rois["labels"], dtype=object).reshape(-1, 1), - dtype=h5py.string_dtype(encoding="utf-8"), - ) - roi_group.create_dataset("F", data=rois["F"]) + rois = assembled["user_rois"] + roi_group = group.create_group("user_rois") + roi_group.attrs["annotation_enabled"] = bool(path.annotation_enabled) + for key in ("mask", "F", "Fsvd"): + roi_group.create_dataset(key, data=rois[key]) + roi_group.create_dataset( + "labels", + data=np.asarray(rois["labels"], dtype=object).reshape(-1, 1), + dtype=h5py.string_dtype(encoding="utf-8"), + ) def write_summary(summary: ExperimentSummary, path: Union[str, Path]) -> None: """Persist models, retaining existing params, flags and unrelated paths. Path identity comes exclusively from PathSummary.path_index. Required - source/ROI arrays are validated before replacing each existing path group. + source/ROI arrays and supplied optional trace shapes are validated before + replacing each existing path group. Disabled annotations still emit empty + ROI datasets; their state is recorded in a group attribute. """ indexes = [item.path_index for item in summary.paths] if len(set(indexes)) != len(indexes): @@ -233,12 +313,26 @@ def read_summary(path: Union[str, Path]) -> ExperimentSummary: Unrecognized root groups are ignored. Paths are sorted by their numeric identity, not lexical order. This is not a general standard-SILo codec. + Legacy missing optional traces remain None; without annotation metadata, + ROI group presence retains its historical enabled/disabled meaning. """ raw = load_struct_h5(path) names = sorted( (key for key in raw if key.startswith("Path") and key[4:].isdigit()), key=lambda key: int(key[4:]), ) + # The shared struct loader intentionally ignores attributes. Legacy files + # used group presence alone to distinguish disabled/empty annotations. + with h5py.File(path, "r") as f: + annotation_enabled = { + name: ( + bool(f[name]["user_rois"].attrs.get( + "annotation_enabled", True + )) + if "user_rois" in f[name] else False + ) + for name in names + } paths = [] for name in names: group = raw[name] @@ -252,6 +346,7 @@ def read_summary(path: Union[str, Path]) -> ExperimentSummary: **group["visualizations"], "global_f": group["global"]["F"], "user_rois": rois, + "annotation_enabled": annotation_enabled[name], } paths.append(_decode_path(assembled, int(name[4:]) - 1)) return ExperimentSummary(paths=paths, params=raw.get("params")) diff --git a/src/giant_python/models/experiment_summary.py b/src/giant_python/models/experiment_summary.py index 1b8f6de..6ca4ddb 100644 --- a/src/giant_python/models/experiment_summary.py +++ b/src/giant_python/models/experiment_summary.py @@ -22,9 +22,10 @@ class Source: coords : ndarray or None 0-indexed ``[z, y, x]`` location. df_ls, df_denoised, events : ndarray or None - Extracted dF (least squares or profile-weighted average, after baseline - re-estimation), denoised dF, and deconvolved events, shaped - ``(channels, frames)``. The historical ``df_ls`` name is retained. + Least-squares dF after baseline re-estimation, the fitted convolution + trace (without another baseline subtraction), and nonnegative event + amplitudes, shaped ``(channels, frames)``. Missing observations and + unprocessed channels are NaN. Events exclude each trial's initial state. f0 : ndarray or None Per-channel baseline. snr : float or None @@ -70,7 +71,8 @@ class UserRoi: Rotation angle (ellipse ROIs). fsvd : ndarray or None ROI signal after SVD / projection, shaped ``(channels, total frames)`` - (``experiment_summary.h5`` ``user_rois/Fsvd``). + (``experiment_summary.h5`` ``user_rois/Fsvd``). Band extraction does + not compute this signal; the writer uses NaN when unavailable. f : ndarray or None Raw or baseline-corrected ROI fluorescence, shaped ``(channels, total frames)`` @@ -142,8 +144,8 @@ class PathSummary: """All eager output for one imaging path, identified by a 0-based index. Arrays remain NumPy-backed in memory; Source/UserRoi entries may be views - of assembled arrays. ``annotation_enabled`` distinguishes an absent ROI - group from an enabled-but-empty group. Private empty-array templates + of assembled arrays. ``annotation_enabled`` distinguishes disabled from + enabled-but-empty annotations (saved as an ROI group attribute). Private empty-array templates retain dimensions and dtypes that cannot be inferred from empty lists; they never duplicate populated source or ROI tensors. """ diff --git a/src/giant_python/models/params.py b/src/giant_python/models/params.py index fd6070d..78d7cb7 100644 --- a/src/giant_python/models/params.py +++ b/src/giant_python/models/params.py @@ -27,6 +27,15 @@ def _positive(name: str, value: object, *, integer: bool = False) -> None: raise ValueError(f"{name} must be a {label}") +def _nonnegative(name: str, value: object) -> None: + """Reject nonfinite, negative, boolean, or nonnumeric values.""" + if ( + isinstance(value, bool) or not isinstance(value, Real) + or not math.isfinite(value) or value < 0 + ): + raise ValueError(f"{name} must be a finite nonnegative number") + + @dataclass class BandSiloParams: """Scientific settings for the implemented SLAP2 band algorithm. @@ -36,7 +45,8 @@ class BandSiloParams: analyze_hz : float Temporal analysis rate in Hz. decay_tau_s : float - Calcium-decay time constant in seconds. + Decay time constant in seconds, shared by localization filtering and + convolution-constrained trace extraction. Saved as decayTau_s. baseline_window_s, denoise_window_s : float Baseline and temporal denoising windows in seconds. vif : float @@ -56,10 +66,16 @@ class BandSiloParams: z_tol : float Positive z tolerance in microns: maximum absolute motion-bin z deviation from the median frame z for low-resolution frame selection. - simple_trace_extraction : bool - Use normalized spatial-profile weighted averages of superpixels for - final dF and background traces instead of least squares. Source - localization is unchanged. Defaults to False. + trace_beta_fraction : float + Nonnegative event penalty relative to each trial's adjoint scale. + trace_max_iter : int + Maximum L-BFGS-B iterations per temporal optimization stage. + trace_tolerance : float + Positive optimizer objective/gradient tolerance. + trace_debias : bool + Refit selected events without L1 shrinkage (default False). + trace_event_threshold : float + Absolute event amplitude threshold for optional debias support. Notes ----- @@ -67,7 +83,8 @@ class BandSiloParams: separate from science. Numerical defaults preserve the band algorithm. Sparse-superpixel background interpolation always uses cubic splines. Channel count is inferred from acquisition metadata; source activity - extraction always uses channel zero. + extraction always uses channel zero. Production extraction always saves + least-squares dF_ls alongside convolution dF_denoised and events. """ analyze_hz: float = 100.0 @@ -81,7 +98,11 @@ class BandSiloParams: peak_buffer: int = 3 psf_dilation: int = 17 z_tol: float = 1.5 - simple_trace_extraction: bool = False + trace_beta_fraction: float = 0.01 + trace_max_iter: int = 500 + trace_tolerance: float = 1e-7 + trace_debias: bool = False + trace_event_threshold: float = 1e-6 def __post_init__(self) -> None: """Validate values supported by the existing numerical path.""" @@ -94,12 +115,15 @@ def __post_init__(self) -> None: "sparse_fac", "peakth", "z_tol", + "trace_tolerance", ): _positive(name, getattr(self, name)) - for name in ("d_xy", "peak_buffer", "psf_dilation"): + for name in ("d_xy", "peak_buffer", "psf_dilation", "trace_max_iter"): _positive(name, getattr(self, name), integer=True) - if not isinstance(self.simple_trace_extraction, bool): - raise ValueError("simple_trace_extraction must be a bool") + if not isinstance(self.trace_debias, bool): + raise ValueError("trace_debias must be a bool") + for name in ("trace_beta_fraction", "trace_event_threshold"): + _nonnegative(name, getattr(self, name)) @dataclass diff --git a/tests/test_band_workflow.py b/tests/test_band_workflow.py index 62e7d43..2cdbb4e 100644 --- a/tests/test_band_workflow.py +++ b/tests/test_band_workflow.py @@ -17,6 +17,7 @@ from giant_python.extraction.band import traces from giant_python.extraction.band import workflow as wf +from giant_python.extraction.band.transient import TransientOptions from giant_python.extraction.band.types import TrialTraceResult from giant_python.io.experiment_summary import read_summary from giant_python.models.params import ( @@ -25,13 +26,18 @@ ExecutionOptions, ) from giant_python.models.trial_table import Slap2Info, TrialTable +from giant_python.numerics.baseline import assemble_dff from giant_python.pipeline.extract import extract_band_sources class TestBandWorkflow(unittest.TestCase): """Exercise the public API through the driver to saved results.""" - def _run_mock_session(self, *, loaded=False, rois=False, simple=False): + def test_default_trace_workflow_and_saved_parameters(self): + """Without mode flags, both real solvers feed assembly and HDF5.""" + self._run_mock_session() + + def _run_mock_session(self, *, loaded=False, rois=False): """Check schema, policy and motion values in a two-path session.""" n_frames, side, n_sp = 120, 15, 49 positions = [(r, c) for r in range(4, 11) for c in range(4, 11)] @@ -46,12 +52,12 @@ def _run_mock_session(self, *, loaded=False, rois=False, simple=False): psf = np.ones((3, 3), dtype=np.float32) options = ExecutionOptions(max_workers=3, max_trials=2) annotation = AnnotationOptions(enabled=rois, interactive=False) - science_args = dict( + params = BandSiloParams( analyze_hz=10, denoise_window_s=0.3, baseline_window_s=0.5, z_tol=2.5, - simple_trace_extraction=simple, + decay_tau_s=.2, trace_beta_fraction=.001, + trace_max_iter=1500, ) - params = BandSiloParams(**science_args) before = (asdict(params), asdict(options), asdict(annotation)) with TemporaryDirectory() as directory, ExitStack() as stack: @@ -293,6 +299,25 @@ def read_trial( ) ) + fit_results = [] + original_fit = traces.fit_transient_traces + + def fit_trial(*args, **kwargs): + """Retain the real fit payload to verify independent assembly.""" + result = original_fit(*args, **kwargs) + fit_results.append(result) + return result + + transient_fit = stack.enter_context( + patch.object(traces, "fit_transient_traces", side_effect=fit_trial) + ) + trace_calls = stack.enter_context( + patch.object(wf, "get_high_res_traces", wraps=wf.get_high_res_traces) + ) + stack.enter_context(patch.object( + traces, "_weighted_trace_average", + side_effect=AssertionError("Production must use QR, not simple averages"), + )) input_value = table if loaded else source summary = extract_band_sources( input=input_value, @@ -320,6 +345,20 @@ def read_trial( for call in select_bins.call_args_list: self.assertEqual(call.kwargs["z_tol"], params.z_tol) self.assertEqual(high_read.call_count, 2) + self.assertEqual(transient_fit.call_count, 2) + self.assertEqual(trace_calls.call_count, 4) + for call in trace_calls.call_args_list: + self.assertNotIn("simple_trace_extraction", call.kwargs) + self.assertNotIn("convolution_trace_extraction", call.kwargs) + self.assertIsInstance(call.kwargs["transient_options"], TransientOptions) + for call in transient_fit.call_args_list: + self.assertEqual(call.kwargs["options"].tau_s, params.decay_tau_s) + self.assertEqual(call.kwargs["options"].beta_fraction, params.trace_beta_fraction) + self.assertEqual(call.kwargs["options"].max_iter, params.trace_max_iter) + self.assertEqual(call.kwargs["options"].tolerance, params.trace_tolerance) + self.assertEqual(call.kwargs["options"].debias, params.trace_debias) + self.assertEqual(call.kwargs["options"].event_threshold, params.trace_event_threshold) + self.assertEqual(call.args[4], params.analyze_hz) self.assertEqual(bin_trial_motion.call_count, 2) for call in bin_trial_motion.call_args_list: self.assertEqual(call.kwargs["z_tol"], params.z_tol) @@ -346,8 +385,14 @@ def read_trial( assert summary.params is not None assert restored.params is not None self.assertEqual(summary.params["numChannels"], 2) - self.assertEqual(summary.params["simple_trace_extraction"], simple) - self.assertEqual(restored.params["simple_trace_extraction"], simple) + for name in ("simple_trace_extraction", "convolution_trace_extraction"): + self.assertFalse(hasattr(params, name)) + self.assertNotIn(name, summary.params) + self.assertNotIn(name, restored.params) + self.assertEqual(restored.params["decayTau_s"], .2) + self.assertNotIn("trace_tau_s", restored.params) + self.assertEqual(restored.params["trace_beta_fraction"], .001) + self.assertEqual(restored.params["trace_max_iter"], 1500) self.assertNotIn("operator", summary.params) self.assertNotIn("operator", restored.params) self.assertEqual(summary.params["draw_user_rois"], rois) @@ -359,6 +404,27 @@ def read_trial( self.assertIsInstance( runtime_result.trace_results[0], TrialTraceResult ) + trial = runtime_result.trace_results[0] + self.assertEqual(len(trial), 10) + fit = fit_results[dmd_ix] + np.testing.assert_allclose(trial.d_f_denoised, fit.d_f, rtol=1e-6) + np.testing.assert_allclose(trial.events, fit.events, rtol=1e-6) + np.testing.assert_array_equal(trial.events[0], 0) + # Independently check the original QR coefficients, rather + # than comparing two aliases of the convolution result. + fit_args = transient_fit.call_args_list[dmd_ix].args + residual, designs, bins, valid = fit_args[:4] + self.assertTrue(valid.all()) + for motion, design in designs.items(): + selected = bins == motion + expected_ls = np.linalg.lstsq( + design, residual[:, selected], rcond=None + )[0].T + expected_f0 = np.linalg.lstsq( + design, 3 * residual[:, selected], rcond=None + )[0].T + np.testing.assert_allclose(trial.d_f[selected], expected_ls, rtol=1e-5) + np.testing.assert_allclose(trial.f0_ls[selected], expected_f0, rtol=1e-5) self.assertEqual(path.path_index, dmd_ix) np.testing.assert_array_equal(path.z_depths, [[10 + dmd_ix]]) np.testing.assert_array_equal( @@ -388,27 +454,23 @@ def read_trial( saved.sources[0].df_ls, equal_nan=True, ) - if simple: - # Constant activity: averaging must preserve its scale, - # regardless of the projected profile's non-unit mass. - np.testing.assert_allclose( - runtime_result.trace_results[0].d_f, - 0.25 * (10 + 5 * dmd_ix), rtol=1e-6, - ) - np.testing.assert_allclose( - runtime_result.trace_results[0].f0_ls, - 0.75 * (10 + 5 * dmd_ix), rtol=1e-6, - ) - np.testing.assert_allclose( - saved.sources[0].df_ls[0], - 0, atol=1e-6, - ) - # Existing baseline re-estimation removes a constant - # signal but preserves total weighted fluorescence. - np.testing.assert_allclose( - saved.sources[0].df_ls[0] + saved.sources[0].f0[0], - 10 + 5 * dmd_ix, rtol=1e-6, - ) + _, expected_f0, expected_ls, _ = assemble_dff( + trial.d_f, trial.f0_ls, + runtime_result.denoise_window, runtime_result.baseline_window, + ) + for source in (path.sources[0], saved.sources[0]): + for field, expected_trace in ( + ("df_ls", expected_ls), ("f0", expected_f0), + ("df_denoised", trial.d_f_denoised), + ("events", trial.events), + ): + values = getattr(source, field) + self.assertEqual(values.shape, (2, n_frames)) + np.testing.assert_array_equal(values[0], expected_trace[:, 0]) + self.assertTrue(np.isnan(values[1]).all()) + self.assertFalse(np.allclose( + source.df_ls, source.df_denoised, equal_nan=True + )) self.assertEqual(len(path.user_rois), int(rois)) self.assertEqual(saved.annotation_enabled, rois) if rois: @@ -431,12 +493,13 @@ def test_loaded_table_is_not_reopened(self): """Prepare a loaded model directly without a second file read.""" self._run_mock_session(loaded=True) - def test_simple_extraction_reaches_saved_traces(self): - """Opt-in averaging reaches per-trial extraction and HDF5 output.""" + def test_default_production_uses_qr_and_convolution(self): + """Production uses QR and fits convolution traces without opt-in.""" with patch.object( - traces, "solve_phi_motion", side_effect=AssertionError("LS called") - ): - self._run_mock_session(loaded=True, simple=True) + traces, "solve_phi_motion", wraps=traces.solve_phi_motion + ) as qr: + self._run_mock_session(loaded=True) + self.assertGreater(qr.call_count, 0) def test_split_policy_preserves_rois_and_caller_options(self): """Split policy leaves scientific options unchanged.""" diff --git a/tests/test_bandsilo_pipeline.py b/tests/test_bandsilo_pipeline.py index e2e46d2..b0f9e8d 100644 --- a/tests/test_bandsilo_pipeline.py +++ b/tests/test_bandsilo_pipeline.py @@ -30,6 +30,7 @@ BandSiloParams, ExecutionOptions, ) +from giant_python.numerics.baseline import assemble_dff def _trial(n_frames, n_sources, n_channels, n_soma): @@ -288,6 +289,24 @@ def test_persisted_background_interpolation_is_cubic(self): ) self.assertEqual(metadata["backgroundInterpolation"], "cubic") + def test_persisted_trace_settings_exclude_removed_mode_keys(self): + """Default metadata retains tuning settings but no trace opt-outs.""" + params = BandSiloParams() + metadata = wf._params_dict( + params, + ResolvedAcquisition(num_channels=1, align_hz=100.0), + AnnotationOptions(), + ) + for name in ("simple_trace_extraction", "convolution_trace_extraction"): + self.assertNotIn(name, metadata) + self.assertEqual(metadata["decayTau_s"], params.decay_tau_s) + for name in ( + "trace_beta_fraction", "trace_max_iter", "trace_tolerance", + "trace_debias", "trace_event_threshold", + ): + with self.subTest(name=name): + self.assertEqual(metadata[name], getattr(params, name)) + def test_default_execution_skips_empty_planes_and_preserves_grid(self): """Interpolate selected planes; baseline and assembly share data.""" data = np.array([[3.0, 4.0], [5.0, 6.0]], dtype=np.float32) @@ -470,6 +489,9 @@ def test_shapes(self): self.assertEqual(source.coords.shape, (3,)) self.assertEqual(source.df_ls.shape, (1, 13)) self.assertEqual(source.f0.shape, (1, 13)) + for values in (source.df_denoised, source.events): + self.assertEqual(values.shape, source.df_ls.shape) + self.assertTrue(np.isnan(values).all()) self.assertEqual(source.df_ls.dtype, np.float64) self.assertEqual(source.f0.dtype, np.float64) np.testing.assert_array_equal([s.snr for s in path.sources], [1, 2]) @@ -487,8 +509,141 @@ def test_two_channels_global_shape(self): pr = _path_result(n_channels=2) path = pl.assemble_path_summary(pr) self.assertEqual(path.global_f.shape, (2, 13)) - self.assertEqual(path.sources[0].df_ls.shape, (2, 13)) - self.assertTrue(np.isnan(path.sources[0].df_ls[1]).all()) + for field in ("df_ls", "f0", "df_denoised", "events"): + values = getattr(path.sources[0], field) + self.assertEqual(values.shape, (2, 13)) + self.assertTrue(np.isnan(values[1]).all()) + + def test_independent_traces_stitch_skips_masks_and_legacy_trials(self): + """Only LS gets baseline processing; raw fit outputs keep trial order.""" + for channels in (1, 2): + with self.subTest(channels=channels): + pr = _path_result(n_sources=2, n_channels=channels) + fitted = [] + for index, trial in enumerate(pr.trace_results): + signal = trial.d_f.copy() - 0.5 + signal[1] = np.nan + denoised = ( + np.arange(signal.size, dtype=np.float32) + .reshape(signal.shape) + 100 * (index + 1) + ) + events = np.zeros_like(signal) + events[2, index] = 7 + index + denoised[1] = np.nan + events[1] = np.nan + # Missing transient values must not alter the LS mask. + denoised[3, 0] = np.nan + events[4, 1] = np.nan + fitted.append(trial._replace( + d_f=signal, d_f_denoised=denoised, events=events, + frame_line_idxs=trial.frame_line_idxs + 100 * index, + )) + skipped = _trial(0, 2, channels, 0)._replace( + d_f_denoised=np.empty((0, 2), dtype=np.float32), + events=np.empty((0, 2), dtype=np.float32), + ) + legacy = _trial(5, 2, channels, 0) + pr.trace_results = [fitted[0], skipped, fitted[1], legacy] + snapshots = [ + tuple(None if value is None else value.copy() for value in ( + trial.d_f, trial.f0_ls, trial.d_f_denoised, trial.events + )) + for trial in pr.trace_results + ] + raw_ls = np.concatenate([r.d_f for r in pr.trace_results]) + raw_f0 = np.concatenate([r.f0_ls for r in pr.trace_results]) + full, expected_f0, expected_ls, _ = assemble_dff( + raw_ls, raw_f0, pr.denoise_window, pr.baseline_window + ) + expected_denoised = np.concatenate([ + fitted[0].d_f_denoised, fitted[1].d_f_denoised, + np.full_like(legacy.d_f, np.nan), + ]) + expected_events = np.concatenate([ + fitted[0].events, fitted[1].events, + np.full_like(legacy.d_f, np.nan), + ]) + with patch.object(pl, "assemble_dff", wraps=assemble_dff) as baseline: + path = pl.assemble_path_summary(pr) + baseline.assert_called_once() + np.testing.assert_array_equal(baseline.call_args.args[0], raw_ls) + np.testing.assert_array_equal(baseline.call_args.args[1], raw_f0) + self.assertEqual(baseline.call_args.args[2:], (3, 5)) + self.assertFalse(np.allclose(raw_ls, expected_ls, equal_nan=True)) + for index, source in enumerate(path.sources): + for field, expected in ( + ("df_ls", expected_ls), ("f0", expected_f0), + ("df_denoised", expected_denoised), + ("events", expected_events), + ): + actual = getattr(source, field) + self.assertEqual(actual.shape, (channels, 18)) + np.testing.assert_array_equal(actual[0], expected[:, index]) + self.assertTrue(np.isnan(actual[1:]).all()) + self.assertEqual(source.df_denoised.dtype, np.float32) + self.assertEqual(source.events.dtype, np.float32) + np.testing.assert_array_equal( + path.frame_info.trial_num_frames.ravel(), [6, 0, 7, 5] + ) + np.testing.assert_array_equal( + path.frame_info.discard_frames.ravel(), + np.any(np.isnan(full), axis=1), + ) + self.assertFalse(path.frame_info.discard_frames[[3, 4, 9, 10]].any()) + np.testing.assert_array_equal( + path.frame_info.frame_line_idxs.ravel(), + np.concatenate([r.frame_line_idxs for r in pr.trace_results]), + ) + np.testing.assert_array_equal( + path.global_f, + np.concatenate([r.global_f for r in pr.trace_results]).T, + ) + for trial, before in zip(pr.trace_results, snapshots): + for actual, expected in zip( + (trial.d_f, trial.f0_ls, trial.d_f_denoised, trial.events), + before, + ): + if expected is None: + self.assertIsNone(actual) + else: + np.testing.assert_array_equal(actual, expected) + + # Changing only the fit cannot change LS, F0 or discard flags. + pr.trace_results = [r._replace( + d_f_denoised=(None if r.d_f_denoised is None + else r.d_f_denoised + 500), + events=(None if r.events is None else r.events * 3), + ) for r in pr.trace_results] + changed = pl.assemble_path_summary(pr) + for before, after in zip(path.sources, changed.sources): + np.testing.assert_array_equal(after.df_ls, before.df_ls) + np.testing.assert_array_equal(after.f0, before.f0) + np.testing.assert_array_equal( + after.df_denoised[0], before.df_denoised[0] + 500 + ) + np.testing.assert_array_equal(after.events[0], before.events[0] * 3) + np.testing.assert_array_equal( + changed.frame_info.discard_frames, path.frame_info.discard_frames + ) + + def test_missing_optional_fields_are_filled_independently(self): + """One absent fit field never erases or substitutes the other field.""" + pr = _path_result(n_channels=2) + first, second = pr.trace_results + events = np.full_like(first.d_f, 7) + denoised = np.full_like(second.d_f, 50) + pr.trace_results = [ + first._replace(events=events), + second._replace(d_f_denoised=denoised), + ] + path = pl.assemble_path_summary(pr) + for index, source in enumerate(path.sources): + self.assertTrue(np.isnan(source.df_denoised[0, :6]).all()) + np.testing.assert_array_equal(source.df_denoised[0, 6:], denoised[:, index]) + np.testing.assert_array_equal(source.events[0, :6], events[:, index]) + self.assertTrue(np.isnan(source.events[0, 6:]).all()) + self.assertTrue(np.isnan(source.df_denoised[1]).all()) + self.assertTrue(np.isnan(source.events[1]).all()) def test_user_rois_assembled(self): """draw_user_rois with masks assembles the user-ROI datasets.""" @@ -533,6 +688,9 @@ def test_empty_sources_preserve_geometry_and_frame_data(self): path._empty_source_arrays["profiles"].shape, (0, 1, 4, 5) ) self.assertEqual(path._empty_source_arrays["dF_ls"].shape, (0, 2, 13)) + for key in ("dF_denoised", "events"): + self.assertEqual(path._empty_source_arrays[key].shape, (0, 2, 13)) + self.assertEqual(path._empty_source_arrays[key].dtype, np.float32) self.assertEqual(path._empty_source_arrays["SNR"].dtype, np.float32) self.assertEqual(path.global_f.shape, (2, 13)) self.assertFalse(path.frame_info.discard_frames.any()) @@ -571,7 +729,17 @@ def test_schema_and_values(self): for key in FRAME_INFO_KEYS: self.assertIn(key, grp["frame_info"]) self.assertIn("act_im", grp["visualizations"]) - self.assertNotIn("user_rois", grp) + for key in ("dF_denoised", "events"): + values = grp[f"sources/temporal/{key}"][()] + self.assertEqual(values.shape, (2, 1, 13)) + self.assertTrue(np.isnan(values).all()) + rois = grp["user_rois"] + self.assertFalse(rois.attrs["annotation_enabled"]) + self.assertEqual(set(rois), {"mask", "labels", "F", "Fsvd"}) + self.assertEqual(rois["mask"].shape, (0, 1, 4, 5)) + self.assertEqual(rois["labels"].shape, (0, 1)) + self.assertEqual(rois["F"].shape, (0, 1, 13)) + self.assertEqual(rois["Fsvd"].shape, (0, 1, 13)) finally: os.remove(path) @@ -591,6 +759,9 @@ def test_user_rois_written(self): self.assertEqual(ur["mask"].shape, (1, 1, 4, 5)) self.assertEqual(ur["F"].shape, (1, 1, 13)) self.assertEqual(ur["labels"].shape, (1, 1)) + self.assertTrue(ur.attrs["annotation_enabled"]) + self.assertEqual(ur["Fsvd"].shape, ur["F"].shape) + self.assertTrue(np.isnan(ur["Fsvd"][()]).all()) finally: os.remove(path) diff --git a/tests/test_bandsilo_traces.py b/tests/test_bandsilo_traces.py index 43afdbd..2587051 100644 --- a/tests/test_bandsilo_traces.py +++ b/tests/test_bandsilo_traces.py @@ -16,6 +16,7 @@ from giant_python.extraction.band import operators from giant_python.extraction.band import traces as tr from giant_python.extraction.band.types import TrialTraceResult +from giant_python.extraction.band.transient import TransientOptions, TransientResult def _geometry(npc=15, npr=15, num_fast_zs=2): @@ -116,6 +117,35 @@ def _compute(inp, **kwargs): ) +class TestTrialTraceResult(unittest.TestCase): + """The original eight positional fields retain their identity and order.""" + + def test_legacy_constructor_defaults_and_extended_fields(self): + original = _compute(_trial_inputs()) + legacy = TrialTraceResult(*original[:8]) + self.assertEqual(len(legacy), 10) + self.assertEqual(legacy._fields, ( + "d_f", "f0_ls", "frame_line_idxs", "selected_pixels", "global_f", + "reference_offsets", "online_shifts", "user_roi_f", + "d_f_denoised", "events", + )) + self.assertIsNone(legacy.d_f_denoised) + self.assertIsNone(legacy.events) + denoised = np.full_like(legacy.d_f, 13) + events = np.full_like(legacy.d_f, 2) + partial = TrialTraceResult(*legacy[:8], denoised) + self.assertIs(partial.d_f_denoised, denoised) + self.assertIsNone(partial.events) + extended = TrialTraceResult(*legacy[:8], denoised, events) + for index, value in enumerate(original[:8]): + self.assertIs(legacy[index], value) + self.assertIs(extended[index], value) + self.assertIs(extended.d_f_denoised, denoised) + self.assertIs(extended.events, events) + self.assertIs(extended[8], denoised) + self.assertIs(extended[9], events) + + class TestWeightedTraceAverage(unittest.TestCase): """Simple traces use profile intensities, not a regression coefficient.""" @@ -195,6 +225,75 @@ def test_motion_projection_background_and_dropped_frames(self): class TestLeastSquaresTraceExtraction(unittest.TestCase): """Missing high-resolution observations must not abort a trial.""" + def test_transient_outputs_never_replace_qr_and_default_return_stays_pair(self): + """Distinct fit payloads stay separate, including their missing masks.""" + profiles = torch.eye(2) + h_mots = [torch.eye(2).to_sparse(), (2 * torch.eye(2)).to_sparse()] + data = np.array([ + [-4., 2., 10., 4., 5., np.inf, 14., 8.], + [3., np.nan, 6., 8., 1., 3., 10., 4.], + ], dtype=np.float32) + background = np.ones_like(data) + background[0, 3] = np.nan + bins = np.array([0, 0, 1, 1, -1, 0, 1, 0]) + valid = np.array([True, False, True, False, False, False, True, True]) + denoised = np.arange(16, dtype=np.float32).reshape(8, 2) + 100 + denoised[~valid] = np.nan + latent_events = np.arange(16, dtype=np.float32).reshape(8, 2) + 20 + latent_events[0] = 0 # Initial state is not a new event. + expected_events = latent_events.copy() + expected_events[~valid] = np.nan + args = ( + data, background, np.array([[0, 0], [1, 0]]), bins, + np.array([0, 1]), np.empty((0, 2)), np.empty(0), np.arange(2), + profiles, 2, 2, + ) + options = TransientOptions() + + def fitted(*args, **kwargs): + """Return fresh arrays so masking cannot affect later calls.""" + return TransientResult( + d_f=denoised.copy(), events=latent_events.copy(), + initial_state=np.array([100., 101.]), + model_trace=np.full((8, 2), 999., dtype=np.float32), + metadata={"converged": True}, + ) + + with patch.object(tr, "build_motion_h_matrices", return_value=h_mots): + qr = tr._solve_trial_phi_f0(*args) + no_fit = tr._solve_trial_phi_f0(*args, return_transient=True) + with patch.object(tr, "fit_transient_traces", side_effect=fitted) as fit: + pair = tr._solve_trial_phi_f0( + *args, transient_options=options, analyze_hz=50, + ) + extended = tr._solve_trial_phi_f0( + *args, transient_options=options, analyze_hz=50, + return_transient=True, + ) + self.assertEqual(len(qr), 2) + self.assertEqual(len(pair), 2) + self.assertEqual(len(extended), 4) + self.assertEqual(len(no_fit), 4) + self.assertEqual(fit.call_count, 2) + for result in (pair, extended, no_fit): + for actual, expected in zip(result[:2], qr): + torch.testing.assert_close(actual, expected, equal_nan=True) + np.testing.assert_array_equal(qr[0][0], [-5., 2.]) + np.testing.assert_array_equal(qr[0][2], [4.5, 2.5]) + np.testing.assert_array_equal(qr[1][[1, 5]], np.ones((2, 2))) + np.testing.assert_array_equal(extended[2], denoised) + np.testing.assert_array_equal(extended[3], expected_events) + self.assertTrue(torch.isnan(no_fit[2]).all()) + self.assertTrue(torch.isnan(no_fit[3]).all()) + for call in fit.call_args_list: + np.testing.assert_array_equal(call.args[0], data - background) + np.testing.assert_array_equal(call.args[1][0], np.eye(2)) + np.testing.assert_array_equal(call.args[1][1], 2 * np.eye(2)) + np.testing.assert_array_equal(call.args[2], bins) + np.testing.assert_array_equal(call.args[3], valid) + self.assertEqual(call.args[4], 50) + self.assertIs(call.kwargs["options"], options) + def test_missing_data_and_background_in_motion_bins(self): """dF and F0 retain their own missing-frame masks across motion bins.""" profiles = torch.tensor([[1., 2.], [3., 0.], [0., 4.]]) @@ -251,15 +350,101 @@ def test_z_tolerance_and_inclusive_boundaries(self): class TestComputeHighResTraces(unittest.TestCase): """compute_high_res_traces per-trial numerics.""" + def test_transient_preserves_qr_masks_geometry_and_auxiliary_outputs(self): + """A real fit adds denoised/events without replacing any legacy field.""" + inp = _trial_inputs(num_channels=2) + inp["a_data"]["motionDSz"][20:30] = 5 + inp["data"][0, 4] = np.nan + inp["data"][1, 5] = np.inf + inp["background_ds"][0, 0:4] = np.nan + inp["frames"] = np.ceil(inp["frames"]) + baseline = _compute(inp) + spatial_before = inp["a_final"].clone() + options = TransientOptions(max_iter=1500) + fitted = [] + original_fit = tr.fit_transient_traces + + def fit_trial(*args, **kwargs): + """Capture the real fit before the wrapper masks exposed events.""" + result = original_fit(*args, **kwargs) + fitted.append(result) + return result + + with patch.object(tr, "fit_transient_traces", side_effect=fit_trial) as solve: + out = _compute(inp, transient_options=options, analyze_hz=50) + solve.assert_called_once() + self.assertEqual(solve.call_args.args[4], 50) + self.assertIs(solve.call_args.kwargs["options"], options) + self.assertEqual(len(out), 10) + for actual, expected in zip(out[:8], baseline[:8]): + np.testing.assert_array_equal(actual, expected) + np.testing.assert_array_equal( + out.d_f_denoised, fitted[0].d_f.astype(np.float32) + ) + expected_events = fitted[0].events.astype(np.float32) + expected_events[~solve.call_args.args[3]] = np.nan + np.testing.assert_array_equal(out.events, expected_events) + missing = np.isnan(baseline.d_f) + self.assertTrue(missing.any()) + self.assertTrue((~missing).any()) + for values in (out.d_f_denoised, out.events): + self.assertEqual(values.shape, baseline.d_f.shape) + self.assertEqual(values.dtype, np.float32) + np.testing.assert_array_equal(np.isnan(values), missing) + self.assertTrue((values[~missing] >= 0).all()) + self.assertFalse(np.shares_memory(values, out.d_f)) + self.assertFalse(np.shares_memory(out.d_f_denoised, out.events)) + self.assertFalse(np.allclose(out.d_f, out.d_f_denoised, equal_nan=True)) + torch.testing.assert_close(inp["a_final"], spatial_before) + + def test_transient_all_missing_no_sources_and_failed_fit(self): + """Empty trials/sources are safe; failed fits never become normal output.""" + inp = _trial_inputs() + options = TransientOptions() + inp["data"][:] = np.nan + with patch.object(tr, "fit_transient_traces") as solve: + out = _compute(inp, transient_options=options, analyze_hz=50) + solve.assert_not_called() + self.assertTrue(np.isnan(out.d_f).all()) + self.assertTrue(np.isnan(out.d_f_denoised).all()) + self.assertTrue(np.isnan(out.events).all()) + inp["a_final"] = inp["a_final"][:, :0] + out = _compute(inp, transient_options=options, analyze_hz=50) + self.assertEqual(out.d_f.shape, (40, 0)) + self.assertEqual(out.d_f_denoised.shape, (40, 0)) + self.assertEqual(out.events.shape, (40, 0)) + inp = _trial_inputs() + with self.assertWarns(RuntimeWarning), self.assertRaisesRegex(RuntimeError, "did not converge"): + _compute(inp, transient_options=TransientOptions(max_iter=1), analyze_hz=50) + + def test_transient_requires_explicit_rate_and_ordered_frames(self): + """Line positions must not be interpreted as time in seconds.""" + inp = _trial_inputs() + options = TransientOptions() + with self.assertRaisesRegex(ValueError, "analyze_hz"): + _compute(inp, transient_options=options) + with self.assertRaisesRegex(ValueError, "mutually exclusive"): + _compute(inp, transient_options=options, analyze_hz=50, + simple_trace_extraction=True) + inp["frames"] = inp["frames"][::-1] + with self.assertRaisesRegex(ValueError, "chronological"): + _compute(inp, transient_options=options, analyze_hz=50) + def test_default_remains_least_squares(self): - """Omitting the flag retains the existing solver for every bin.""" + """Isolated low-level extraction still permits QR without a fit.""" inp = _trial_inputs() - with patch.object(tr, "solve_phi_motion", wraps=tr.solve_phi_motion) as ls: + with ( + patch.object(tr, "solve_phi_motion", wraps=tr.solve_phi_motion) as ls, + patch.object(tr, "fit_transient_traces") as fit, + ): default = _compute(inp) + fit.assert_not_called() self.assertGreater(ls.call_count, 0) explicit = _compute(inp, simple_trace_extraction=False) np.testing.assert_array_equal(default.d_f, explicit.d_f) np.testing.assert_array_equal(default.f0_ls, explicit.f0_ls) + self.assertTrue(np.isnan(default.d_f_denoised).all()) + self.assertTrue(np.isnan(default.events).all()) def test_simple_mode_preserves_mask_and_auxiliary_outputs(self): """Only dF/F0 extraction changes; z rejection and other channels stay.""" @@ -283,7 +468,7 @@ def test_simple_mode_preserves_mask_and_auxiliary_outputs(self): ) def test_single_channel_shapes_and_fit(self): - """Returns the 8-tuple; phi/F0 are per-frame per-source with fits.""" + """Returns ten fields; the original eight retain their positions.""" inp = _trial_inputs(num_channels=1) out = _compute(inp) self.assertIsInstance(out, TrialTraceResult) @@ -295,10 +480,18 @@ def test_single_channel_shapes_and_fit(self): self.assertIs(out.reference_offsets, out[5]) self.assertIs(out.online_shifts, out[6]) self.assertIs(out.user_roi_f, out[7]) - phi, f0, frames, sel, global_f, motion, online, f_soma = out + self.assertIs(out.d_f_denoised, out[8]) + self.assertIs(out.events, out[9]) + ( + phi, f0, frames, sel, global_f, motion, online, f_soma, + denoised, events, + ) = out n_sources = inp["a_final"].shape[1] self.assertEqual(phi.shape, (inp["n_frames"], n_sources)) self.assertEqual(f0.shape, (inp["n_frames"], n_sources)) + for values in (denoised, events): + self.assertEqual(values.shape, phi.shape) + self.assertTrue(np.isnan(values).all()) self.assertEqual(frames.shape, (inp["n_frames"],)) self.assertEqual(global_f.shape, (inp["n_frames"], 1)) self.assertEqual( @@ -312,7 +505,7 @@ def test_two_channels(self): """A second channel adds columns to global and soma fluorescence.""" inp = _trial_inputs(num_channels=2) out = _compute(inp) - _, _, _, _, global_f, _, _, f_soma = out + global_f, f_soma = out.global_f, out.user_roi_f self.assertEqual(global_f.shape, (inp["n_frames"], 2)) self.assertEqual( f_soma.shape, (inp["n_frames"], len(inp["soma_sps"]), 2) @@ -333,7 +526,17 @@ class TestGetHighResTraces(unittest.TestCase): """get_high_res_traces IO wrapper (skip + monkeypatched read).""" def test_skipped_trial_returns_empty(self): - """keep_trial=False returns the empty 8-tuple without reading.""" + """keep_trial=False returns all ten empty fields without reading.""" + loader = patch.object( + tr, "_load_high_res_trial_data", side_effect=AssertionError("read skipped trial") + ) + fit = patch.object( + tr, "fit_transient_traces", side_effect=AssertionError("fit skipped trial") + ) + loader.start() + fit.start() + self.addCleanup(loader.stop) + self.addCleanup(fit.stop) inp = _trial_inputs(num_channels=2) g = inp["g"] out = tr.get_high_res_traces( @@ -357,9 +560,13 @@ def test_skipped_trial_returns_empty(self): g["npr"], 2, inp["soma_sps"], + transient_options=TransientOptions(), ) self.assertIsInstance(out, TrialTraceResult) + self.assertEqual(len(out), 10) self.assertEqual(out[0].shape, (0, inp["a_final"].shape[1])) + self.assertEqual(out.d_f_denoised.shape, out.d_f.shape) + self.assertEqual(out.events.shape, out.d_f.shape) self.assertEqual(out[4].shape, (0, 2)) # globalF (0, channels) self.assertEqual(out[2].shape, (0,)) # frames @@ -368,8 +575,9 @@ def test_kept_trial_reads_and_computes(self): for simple in (False, True): with self.subTest(simple_trace_extraction=simple): self._check_kept_trial(simple) + self._check_kept_trial(False, TransientOptions(max_iter=1500)) - def _check_kept_trial(self, simple): + def _check_kept_trial(self, simple, transient_options=None): """Verify IO wrapper forwards the selected extraction mode.""" inp = _trial_inputs(num_channels=1) g = inp["g"] @@ -408,15 +616,21 @@ def fake_load(*args, **kwargs): 1, inp["soma_sps"], simple_trace_extraction=simple, + transient_options=transient_options, ) finally: tr._load_high_res_trial_data = orig # matches a direct compute with the same inputs - expected = _compute(inp, simple_trace_extraction=simple) - np.testing.assert_allclose( - np.nan_to_num(out[0]), np.nan_to_num(expected[0]), rtol=1e-6 + expected = _compute( + inp, simple_trace_extraction=simple, + transient_options=transient_options, analyze_hz=100, ) + for field in ("d_f", "f0_ls", "d_f_denoised", "events"): + np.testing.assert_allclose( + getattr(out, field), getattr(expected, field), + rtol=1e-6, equal_nan=True, + ) self.assertEqual(out[0].shape, (inp["n_frames"], 3)) diff --git a/tests/test_bandsilo_transient.py b/tests/test_bandsilo_transient.py new file mode 100644 index 0000000..4a23d9d --- /dev/null +++ b/tests/test_bandsilo_transient.py @@ -0,0 +1,169 @@ +"""Fixed-profile temporal fitting: motion, gaps, boundaries and gradients.""" + +import unittest +from unittest.mock import patch + +import numpy as np +from scipy.optimize import OptimizeResult + +from giant_python.extraction.band import transient as tt + + +def synthetic_problem(): + """Two overlapping sources, alternating motion and an observation gap.""" + designs = { + 0: np.array([[1, .2], [.1, 1], [.5, .3]], dtype=np.float32), + 1: np.array([[.2, 1], [1, .1], [.4, .8]], dtype=np.float32), + } + events = np.zeros((65, 2)) + events[0] = [.4, .2] + events[15, 0], events[38, 1] = 2, 3 + expected = tt.TransientOperator(50, .15).forward(events) + motion = np.arange(65) % 2 + residual = np.stack([ + designs[int(m)] @ expected[t] for t, m in enumerate(motion) + ], axis=1).astype(np.float32) + valid = np.ones(65, dtype=bool) + valid[9:12] = False + residual[:, 9:12] = np.nan + return residual, designs, motion, valid, expected, events + + +class TestTransientSolver(unittest.TestCase): + """Numerical contracts independent of acquisition files and localization.""" + + def test_adjoint_and_gradient(self): + """Analytic derivatives agree with directional finite differences.""" + rng = np.random.default_rng(11) + s, z = rng.normal(size=(40, 2)), rng.normal(size=(40, 2)) + groups = {0: np.arange(0, 40, 2), 1: np.arange(1, 40, 2)} + normal = {} + for m, ix in groups.items(): + x = rng.normal(size=(3, 2)) + normal[m] = x.T @ x, rng.normal(size=(2, len(ix))) + for kernel in (None, [1, .8, .2]): + op = tt.TransientOperator(50, .15, kernel) + self.assertAlmostEqual( + np.sum(op.forward(s) * z), np.sum(s * op.adjoint(z)), places=10 + ) + _, grad = tt._temporal_quadratic(normal, groups, op.forward(s)) + direction = rng.normal(size=s.shape) + eps = 1e-5 + hi = tt._temporal_quadratic(normal, groups, op.forward(s + eps * direction))[0] + lo = tt._temporal_quadratic(normal, groups, op.forward(s - eps * direction))[0] + self.assertAlmostEqual( + (hi - lo) / (2 * eps), np.sum(direction * op.adjoint(grad)), places=5 + ) + + def test_recovery_masks_initial_state_and_independent_trials(self): + """Motion changes and missing frames never reset the decay state.""" + r, designs, motion, valid, expected, events = synthetic_problem() + options = tt.TransientOptions(beta=0, max_iter=1500, tolerance=1e-12) + result = tt.fit_transient_traces(r, designs, motion, valid, 50, options=options) + self.assertTrue(result.metadata["converged"]) + np.testing.assert_allclose(result.d_f[valid], expected[valid], atol=5e-4) + np.testing.assert_allclose(result.initial_state, events[0], atol=5e-4) + self.assertTrue(np.isnan(result.d_f[~valid]).all()) + self.assertTrue(np.isfinite(result.model_trace).all()) + self.assertTrue((result.events >= 0).all()) + self.assertTrue((result.events[0] == 0).all()) + self.assertEqual(np.argmax(result.events[:, 0]), 15) + self.assertEqual(np.argmax(result.events[:, 1]), 38) + reconstructed = result.events.copy() + reconstructed[0] = result.initial_state + np.testing.assert_array_equal( + result.model_trace, tt.TransientOperator(50, .15).forward(reconstructed) + ) + second = tt.fit_transient_traces( + np.zeros_like(r), designs, motion, np.ones_like(valid), 50, options=options + ) + np.testing.assert_array_equal(second.model_trace, 0) + # Neither caller masks nor input data are modified. + np.testing.assert_array_equal(valid, synthetic_problem()[3]) + np.testing.assert_array_equal(r, synthetic_problem()[0]) + + def test_debias_and_event_penalty(self): + """Optional support refitting restores amplitudes without new events.""" + r, designs, motion, valid, expected, events = synthetic_problem() + settings = dict(beta=.2, max_iter=2000, tolerance=1e-12) + selected = tt.fit_transient_traces( + r, designs, motion, valid, 50, options=tt.TransientOptions(**settings) + ) + result = tt.fit_transient_traces( + r, designs, motion, valid, 50, + options=tt.TransientOptions(**settings, debias=True, event_threshold=.01), + ) + self.assertTrue(result.metadata["converged"]) + self.assertTrue(result.metadata["debias"]["performed"]) + self.assertEqual(result.metadata["final_event_penalty"], 0) + np.testing.assert_allclose(result.d_f[valid], expected[valid], atol=5e-4) + np.testing.assert_allclose(result.events[1:], events[1:], atol=5e-4) + self.assertTrue((result.events[selected.events <= .01] == 0).all()) + + def test_nonconvergence_and_failed_debias(self): + """Both optimization stages expose failures, never silently succeed.""" + r, designs, motion, valid, *_ = synthetic_problem() + options = tt.TransientOptions(max_iter=1, debias=True) + with self.assertWarns(RuntimeWarning): + result = tt.fit_transient_traces(r, designs, motion, valid, 50, options=options) + self.assertFalse(result.metadata["converged"]) + self.assertFalse(result.metadata["debias"]["performed"]) + success = OptimizeResult(x=np.ones(130), fun=0., success=True, nit=2, message="ok") + failed = OptimizeResult(x=np.ones(130), fun=0., success=False, nit=1, message="limit") + with patch.object(tt, "minimize", side_effect=[success, failed]): + with self.assertWarns(RuntimeWarning): + result = tt.fit_transient_traces( + r, designs, motion, valid, 50, + options=tt.TransientOptions(debias=True, event_threshold=0), + ) + self.assertFalse(result.metadata["converged"]) + self.assertTrue(result.metadata["debias"]["performed"]) + + def test_invalid_settings_and_inputs(self): + """Reject malformed arrays, rates and optimization options.""" + for key, values in { + "tau_s": [0, -1, np.nan, True], + "beta_fraction": [-1, np.inf, True], + "beta": [-1, np.nan, True], + "max_iter": [0, 1.5, True], + "tolerance": [0, np.inf], + "debias": [1, "true"], + "event_threshold": [-1, np.nan], + }.items(): + for value in values: + with self.subTest(key=key, value=value), self.assertRaises(ValueError): + tt.TransientOptions(**{key: value}) + r, designs, motion, valid, *_ = synthetic_problem() + for rate in (None, 0, -1, np.nan, True): + with self.assertRaises(ValueError): + tt.fit_transient_traces(r, designs, motion, valid, rate) + for kernel in ([0, 1], [], [-1, 2], [1, np.nan]): + with self.assertRaises(ValueError): + tt.TransientOperator(50, .15, kernel) + for rr, xx, mm, vv in ( + (r.ravel(), designs, motion, valid), + (r, designs, motion[:-1], valid), + (r, {}, motion, valid), + (r, {0: np.ones(3)}, motion, valid), + (r, {0: np.full((3, 2), np.nan)}, motion, valid), + (r, {0: np.zeros((3, 2))}, motion, valid), + (r, designs, motion, np.zeros_like(valid)), + ): + with self.assertRaises(ValueError): + tt.fit_transient_traces(rr, xx, mm, vv, 50) + + def test_rank_deficiency_and_nonfinite_complete_frames(self): + """No inverse is required, and individual bad samples mask whole frames.""" + r, designs, motion, valid, *_ = synthetic_problem() + for x in designs.values(): + x[:, 1] = x[:, 0] + r[0, 1] = np.inf + motion[5] = -1 + result = tt.fit_transient_traces(r, designs, motion, valid, 50) + self.assertTrue(result.metadata["converged"]) + self.assertTrue(np.isnan(result.d_f[[1, 5, 9, 10, 11]]).all()) + self.assertTrue(np.isfinite(result.d_f[[0, 2, 3, 4]]).all()) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_cli.py b/tests/test_cli.py index daed18d..b6635c3 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -97,6 +97,36 @@ def test_supported_routing_choices(self): class TestOptionsFromArgs(unittest.TestCase): """CLI separates science, execution and annotation configuration.""" + def test_defaults_have_no_trace_mode_fields(self): + """Both commands build options without removed mode fields.""" + parser = cli.build_parser() + for command in ("annotate", "extract"): + with self.subTest(command=command): + args = parser.parse_args([command, "tt.h5"]) + original = vars(args).copy() + params, _, _ = cli._options_from_args(args) + for name in ("simple_trace_extraction", "convolution_trace_extraction"): + self.assertFalse(hasattr(args, name)) + self.assertFalse(hasattr(params, name)) + self.assertEqual(vars(args), original) + + def test_trace_tuning_flags(self): + """Trace controls and the shared decay setting reach science options.""" + parser = cli.build_parser() + args = parser.parse_args([ + "extract", "tt.h5", + "--decay-tau-s", "0.25", "--trace-beta-fraction", "0.02", + "--trace-max-iter", "900", "--trace-tolerance", "1e-9", + "--trace-debias", "--trace-event-threshold", "0.001", + ]) + params, _, _ = cli._options_from_args(args) + self.assertEqual(params.decay_tau_s, .25) + self.assertEqual(params.trace_beta_fraction, .02) + self.assertEqual(params.trace_max_iter, 900) + self.assertEqual(params.trace_tolerance, 1e-9) + self.assertTrue(params.trace_debias) + self.assertEqual(params.trace_event_threshold, .001) + def test_maps_fields(self): """Run and annotation flags do not change scientific defaults.""" args = cli.build_parser().parse_args( @@ -113,7 +143,6 @@ def test_maps_fields(self): params, execution, annotations = cli._options_from_args(args) self.assertEqual(params.peakth, 7.0) self.assertEqual(params.z_tol, 1.5) - self.assertFalse(params.simple_trace_extraction) self.assertFalse(hasattr(params, "scan_mode")) self.assertTrue(annotations.enabled) self.assertTrue(annotations.interactive) @@ -210,13 +239,13 @@ def test_extract_service(self): "2.5", "--headless", "--draw-user-rois", - "--simple-trace-extraction", ] ) self.assertEqual(code, 0) self.assertEqual(extract.call_args.args[0], "curated_v2.h5") self.assertEqual(extract.call_args.args[1].z_tol, 2.5) - self.assertTrue(extract.call_args.args[1].simple_trace_extraction) + for name in ("simple_trace_extraction", "convolution_trace_extraction"): + self.assertFalse(hasattr(extract.call_args.args[1], name)) self.assertEqual(extract.call_args.kwargs["scan_mode"], "band") self.assertEqual(extract.call_args.kwargs["execution"].max_trials, 4) self.assertTrue(extract.call_args.kwargs["annotations"].enabled) @@ -255,6 +284,32 @@ def test_removed_commands_fail_before_dispatch(self): annotate.assert_not_called() extract.assert_not_called() + def test_removed_trace_flags_fail_before_dispatch(self): + """Each former flag is unknown, not a compatibility or mode switch.""" + flags = ("--simple-trace-extraction", "--convolution-trace-extraction") + with ( + mock.patch.object(cli, "_cmd_annotate") as annotate, + mock.patch.object(cli, "_cmd_extract") as extract, + ): + for command in ("annotate", "extract"): + for supplied in ([flags[0]], [flags[1]], list(flags)): + with self.subTest(command=command, flags=supplied): + with mock.patch("sys.stderr", new_callable=io.StringIO) as stderr: + with self.assertRaises(SystemExit) as error: + cli.main([command, "tt.h5", *supplied]) + self.assertEqual(error.exception.code, 2) + self.assertIn("unrecognized arguments", stderr.getvalue()) + for flag in supplied: + self.assertIn(flag, stderr.getvalue()) + with mock.patch("sys.stdout", new_callable=io.StringIO) as stdout: + with self.assertRaises(SystemExit) as error: + cli.main([command, "--help"]) + self.assertEqual(error.exception.code, 0) + for flag in flags: + self.assertNotIn(flag, stdout.getvalue()) + annotate.assert_not_called() + extract.assert_not_called() + def test_unsupported_routing_reports_parser_error(self): """Unsupported routing is rejected by argparse before dispatch.""" with ( diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 4b333ac..8642740 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -3,6 +3,7 @@ import math import unittest from copy import deepcopy +from dataclasses import asdict, fields from giant_python.models.params import ( AnnotationOptions, @@ -22,7 +23,11 @@ def test_defaults_match_existing_science(self): self.assertEqual(science.denoise_window_s, 1.0) self.assertEqual(science.vif, 1.38) self.assertEqual(science.z_tol, 1.5) - self.assertFalse(science.simple_trace_extraction) + for name in ("simple_trace_extraction", "convolution_trace_extraction"): + with self.subTest(name=name): + self.assertFalse(hasattr(science, name)) + self.assertNotIn(name, {field.name for field in fields(BandSiloParams)}) + self.assertNotIn(name, asdict(science)) self.assertEqual(science.sparse_fac, math.exp(-3.0)) self.assertEqual(execution, ExecutionOptions(6, False, None)) self.assertEqual(annotations, AnnotationOptions(False, None)) @@ -64,8 +69,7 @@ def test_every_result_is_fresh(self): def test_dictionary_inputs_and_overrides_are_not_modified(self): """Each dictionary affects only its scope and is copied.""" - params = {"analyze_hz": 80.0, "z_tol": 2.5, - "simple_trace_extraction": True} + params = {"analyze_hz": 80.0, "z_tol": 2.5, "trace_debias": True} execution = {"max_workers": 2, "verbose": True} annotations = { "enabled": True, @@ -77,7 +81,7 @@ def test_dictionary_inputs_and_overrides_are_not_modified(self): ) self.assertEqual(science.analyze_hz, 80.0) self.assertEqual(science.z_tol, 2.5) - self.assertTrue(science.simple_trace_extraction) + self.assertTrue(science.trace_debias) self.assertEqual(run, ExecutionOptions(2, True, None)) self.assertEqual(roi, AnnotationOptions(True, False)) self.assertEqual((params, execution, annotations), original) @@ -86,12 +90,58 @@ def test_dictionary_inputs_and_overrides_are_not_modified(self): class TestConfigurationValidation(unittest.TestCase): """Reject unsupported settings before they reach numerical kernels.""" - def test_simple_trace_extraction_requires_bool(self): - """Strings and integers cannot silently select an extraction mode.""" - for value in (0, 1, "true", "false", None): - with self.subTest(value=value): - with self.assertRaisesRegex(ValueError, "simple_trace_extraction"): - resolve_band_options({"simple_trace_extraction": value}) + def test_trace_tuning_options_use_shared_decay(self): + """Trace fitting uses the existing shared decay configuration.""" + params, _, _ = resolve_band_options({ + "decay_tau_s": .3, + "trace_beta_fraction": 0, "trace_max_iter": 1500, + "trace_debias": True, + }) + self.assertEqual(params.decay_tau_s, .3) + self.assertFalse(hasattr(params, "trace_tau_s")) + self.assertEqual(params.trace_beta_fraction, 0) + self.assertTrue(params.trace_debias) + self.assertEqual(params.trace_max_iter, 1500) + for key, values in { + "trace_debias": [1, "true"], + "decay_tau_s": [0, -1, math.nan, True], + "trace_tolerance": [0, math.inf], + "trace_max_iter": [0, 1.5, True], + "trace_beta_fraction": [-1, math.nan, True], + "trace_event_threshold": [-1, math.inf], + }.items(): + for value in values: + with self.subTest(key=key, value=value), self.assertRaisesRegex(ValueError, key): + BandSiloParams(**{key: value}) + + def test_removed_trace_flags_reject_all_boolean_combinations(self): + """Supplying both removed keys fails instead of normalizing them.""" + for simple in (False, True): + for convolution in (False, True): + with self.subTest(simple=simple, convolution=convolution): + supplied: dict[str, bool] = { + "simple_trace_extraction": simple, + "convolution_trace_extraction": convolution, + } + original = supplied.copy() + with self.assertRaisesRegex(TypeError, "trace_extraction"): + BandSiloParams(**supplied) + with self.assertRaisesRegex(TypeError, "trace_extraction"): + resolve_band_options(supplied) + self.assertEqual(supplied, original) + + def test_removed_trace_flags_reject_model_and_dictionary_inputs(self): + """Each removed key is unknown regardless of its supplied value.""" + for name in ("simple_trace_extraction", "convolution_trace_extraction"): + for value in (False, True, 0, 1, "true", "false", None, [], 0.0): + with self.subTest(name=name, value=value): + supplied = {"peakth": 8.0, name: value} + original = deepcopy(supplied) + with self.assertRaisesRegex(TypeError, name): + BandSiloParams(**supplied) + with self.assertRaisesRegex(TypeError, name): + resolve_band_options(supplied) + self.assertEqual(supplied, original) def test_positive_scientific_values(self): """Rates, windows and positive scale values must be finite.""" diff --git a/tests/test_experiment_summary_codec.py b/tests/test_experiment_summary_codec.py new file mode 100644 index 0000000..319d770 --- /dev/null +++ b/tests/test_experiment_summary_codec.py @@ -0,0 +1,405 @@ +"""Focused full-schema summary codec and legacy compatibility coverage.""" + +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory + +import h5py +import numpy as np + +from giant_python.io.experiment_summary import read_summary, write_summary +from giant_python.models.experiment_summary import ( + FRAME_INFO_KEYS, + ExperimentSummary, + FrameInfo, + PathSummary, + Source, + UserRoi, + Visualizations, +) + + +def _path(n_sources=2, n_rois=2, annotation_enabled=True, frames=7): + """Use distinct values and non-square geometry to expose axis changes.""" + trace = np.arange(2 * frames, dtype=np.float32).reshape(2, frames) + profile = np.arange(60, dtype=np.float32).reshape(3, 4, 5) + return PathSummary( + sources=[ + Source( + profile=profile + index, + coords=np.array([index, index + 1, index + 2], np.int32), + df_ls=trace + index, + df_denoised=trace + 100 + index, + events=trace + 200 + index, + f0=trace + 300 + index, + snr=np.float32(index + 0.5), + ) + for index in range(n_sources) + ], + user_rois=[ + UserRoi( + label=f"ROI μ {index}", + mask=(profile % (index + 2)) == 0, + f=trace + 400 + index, + fsvd=trace + 500 + index, + ) + for index in range(n_rois) + ], + visualizations=Visualizations( + mean_im=np.stack([profile, profile + 10]), + act_im=profile + 20, + act_im_peaks=np.zeros((n_sources, 3), np.int32), + ), + frame_info=FrameInfo(**{ + key: (np.arange(frames, dtype=np.float32) - index).reshape(-1, 1) + for index, key in enumerate(FRAME_INFO_KEYS) + }), + global_f=trace + 600, + z_depths=np.array([[2], [4], [8]], dtype=np.int16), + annotation_enabled=annotation_enabled, + ) + + +def _snapshot(destination): + """Capture datasets, groups and attributes without relying on HDF5 bytes.""" + snapshot = {} + with h5py.File(destination, "r") as file: + def collect(name, item): + """Keep values, dtype, shape and all metadata for comparison.""" + snapshot[(name, "kind")] = type(item).__name__ + for key, value in item.attrs.items(): + snapshot[(name, "attribute", key)] = value + if isinstance(item, h5py.Dataset): + snapshot[(name, "value")] = item[()] + snapshot[(name, "dtype")] = item.dtype + snapshot[(name, "shape")] = item.shape + + collect("", file) + file.visititems(collect) + return snapshot + + +class TestExperimentSummaryCodec(unittest.TestCase): + """Optional data stay genuine, missing data stay explicitly unavailable.""" + + def assert_full_schema(self, file, sources, rois, frames=7): + """Verify the full row-major schema, including zero-length axes.""" + group = file["Path1"] + self.assertEqual(set(group), { + "Z_depths", "sources", "frame_info", "global", + "visualizations", "user_rois", + }) + self.assertEqual(group["Z_depths"].shape, (3, 1)) + self.assertEqual(set(group["sources"]), {"spatial", "temporal"}) + spatial = group["sources/spatial"] + self.assertEqual(set(spatial), {"profiles", "coords"}) + self.assertEqual(spatial["profiles"].shape, (sources, 3, 4, 5)) + self.assertEqual(spatial["coords"].shape, (sources, 3)) + temporal = group["sources/temporal"] + self.assertEqual(set(temporal), { + "dF_ls", "dF_denoised", "events", "F0", "SNR", + }) + for key in ("dF_ls", "dF_denoised", "events", "F0"): + self.assertEqual(temporal[key].shape, (sources, 2, frames)) + self.assertEqual(temporal["SNR"].shape, (sources, 1)) + roi_group = group["user_rois"] + self.assertEqual(set(roi_group), {"mask", "labels", "F", "Fsvd"}) + self.assertEqual(roi_group["mask"].shape, (rois, 3, 4, 5)) + self.assertEqual(roi_group["labels"].shape, (rois, 1)) + for key in ("F", "Fsvd"): + self.assertEqual(roi_group[key].shape, (rois, 2, frames)) + self.assertEqual(set(group["frame_info"]), set(FRAME_INFO_KEYS)) + self.assertEqual(set(group["global"]), {"F"}) + self.assertEqual(group["global/F"].shape, (2, frames)) + self.assertEqual(set(group["visualizations"]), { + "mean_im", "act_im", "act_im_peaks", + }) + + def test_supplied_arrays_roundtrip_without_scientific_changes(self): + """All new and existing arrays preserve values, dtype and orientation.""" + original = _path() + original.sources[0].df_denoised[0, 1] = np.nan + original.sources[1].events = original.sources[1].events.astype(np.int16) + original.user_rois[0].fsvd = original.user_rois[0].fsvd.astype(np.float64) + with TemporaryDirectory() as directory: + destination = Path(directory) / "summary.h5" + rewritten = Path(directory) / "rewritten.h5" + write_summary(ExperimentSummary( + paths=[original], params={"operator": "μ", "channels": 2} + ), destination) + with h5py.File(destination, "r") as file: + self.assert_full_schema(file, 2, 2) + self.assertEqual(file["row_major"][()], 1) + self.assertTrue( + file["Path1/user_rois"].attrs["annotation_enabled"] + ) + for dataset, field in ( + ("dF_denoised", "df_denoised"), ("events", "events") + ): + np.testing.assert_array_equal( + file[f"Path1/sources/temporal/{dataset}"][()], + np.stack([getattr(s, field) for s in original.sources]), + ) + np.testing.assert_array_equal( + file["Path1/user_rois/Fsvd"][()], + np.stack([roi.fsvd for roi in original.user_rois]), + ) + summary = read_summary(destination) + loaded = summary.paths[0] + self.assertTrue(loaded.annotation_enabled) + self.assertEqual(summary.params["operator"], "μ") + for before, after in zip(original.sources, loaded.sources): + for field in ( + "profile", "coords", "df_ls", "df_denoised", + "events", "f0", "snr", + ): + np.testing.assert_array_equal( + getattr(after, field), getattr(before, field) + ) + for before, after in zip(original.user_rois, loaded.user_rois): + self.assertEqual(after.label, before.label) + for field in ("mask", "f", "fsvd"): + np.testing.assert_array_equal( + getattr(after, field), getattr(before, field) + ) + for key in FRAME_INFO_KEYS: + np.testing.assert_array_equal( + getattr(loaded.frame_info, key), + getattr(original.frame_info, key), + ) + for field in ("mean_im", "act_im", "act_im_peaks"): + np.testing.assert_array_equal( + getattr(loaded.visualizations, field), + getattr(original.visualizations, field), + ) + np.testing.assert_array_equal(loaded.global_f, original.global_f) + np.testing.assert_array_equal(loaded.z_depths, original.z_depths) + write_summary(summary, rewritten) + before, after = _snapshot(destination), _snapshot(rewritten) + self.assertEqual(before.keys(), after.keys()) + for key in before: + np.testing.assert_equal(after[key], before[key]) + + def test_missing_optional_arrays_are_nan_not_copies_or_zeros(self): + """Mixed missing/present traces are handled independently per entry.""" + for integer_reference in (False, True): + with self.subTest(integer_reference=integer_reference): + original = _path() + original.sources[0].df_denoised = None + original.sources[1].events = None + original.sources[0].snr = None + original.user_rois[0].fsvd = None + if integer_reference: + for source in original.sources: + source.df_ls = source.df_ls.astype(np.int16) + for roi in original.user_rois: + roi.f = roi.f.astype(np.int16) + with TemporaryDirectory() as directory: + destination = Path(directory) / "summary.h5" + write_summary(ExperimentSummary(paths=[original]), destination) + loaded = read_summary(destination).paths[0] + for missing, reference in ( + (loaded.sources[0].df_denoised, loaded.sources[0].df_ls), + (loaded.sources[1].events, loaded.sources[1].df_ls), + (loaded.user_rois[0].fsvd, loaded.user_rois[0].f), + ): + self.assertEqual(missing.shape, reference.shape) + self.assertTrue(np.isnan(missing).all()) + self.assertTrue(np.isnan(loaded.sources[0].snr)) + np.testing.assert_array_equal( + loaded.sources[1].df_denoised, + original.sources[1].df_denoised, + ) + np.testing.assert_array_equal( + loaded.sources[0].events, original.sources[0].events + ) + np.testing.assert_array_equal( + loaded.user_rois[1].fsvd, original.user_rois[1].fsvd + ) + self.assertIsNone(original.sources[0].df_denoised) + self.assertIsNone(original.sources[1].events) + self.assertIsNone(original.user_rois[0].fsvd) + + def test_empty_sources_and_rois_always_have_full_rank(self): + """Geometry inference and legacy templates work for either ROI state.""" + for enabled in (False, True): + for templates in (False, True): + for frames in (0, 7): + with self.subTest( + enabled=enabled, templates=templates, frames=frames + ), TemporaryDirectory() as directory: + original = _path(0, 0, enabled, frames) + if templates: + original._empty_source_arrays = { + "profiles": np.empty((0, 3, 4, 5), np.float64), + "coords": np.empty((0, 3), np.int16), + "dF_ls": np.empty((0, 2, frames), np.float32), + "F0": np.empty((0, 2, frames), np.float32), + "SNR": np.empty((0, 1), np.float32), + } + original._empty_roi_arrays = { + "mask": np.empty((0, 3, 4, 5), bool), + "F": np.empty((0, 2, frames), np.float64), + } + destination = Path(directory) / "summary.h5" + rewritten = Path(directory) / "rewritten.h5" + write_summary( + ExperimentSummary(paths=[original]), destination + ) + loaded = read_summary(destination) + self.assertEqual(loaded.paths[0].sources, []) + self.assertEqual(loaded.paths[0].user_rois, []) + self.assertEqual( + loaded.paths[0].annotation_enabled, enabled + ) + write_summary(loaded, rewritten) + for filename in (destination, rewritten): + with h5py.File(filename, "r") as file: + self.assert_full_schema(file, 0, 0, frames) + self.assertEqual( + bool(file["Path1/user_rois"].attrs[ + "annotation_enabled" + ]), enabled, + ) + if templates: + self.assertEqual( + file["Path1/sources/spatial/profiles"].dtype, + np.dtype(np.float64), + ) + self.assertEqual( + file["Path1/user_rois/F"].dtype, + np.dtype(np.float64), + ) + self.assertNotIn( + "dF_denoised", original._empty_source_arrays + ) + self.assertNotIn("Fsvd", original._empty_roi_arrays) + + def test_legacy_missing_datasets_and_annotation_metadata(self): + """Old absent/present ROI groups preserve their original semantics.""" + for sources in (0, 2): + for state in ("disabled", "empty", "populated"): + with self.subTest(sources=sources, state=state): + rois = 2 if state == "populated" else 0 + original = _path(sources, rois, state != "disabled") + with TemporaryDirectory() as directory: + destination = Path(directory) / "legacy.h5" + rewritten = Path(directory) / "upgraded.h5" + write_summary( + ExperimentSummary(paths=[original]), destination + ) + with h5py.File(destination, "a") as file: + for key in ("dF_denoised", "events"): + del file[f"Path1/sources/temporal/{key}"] + if state == "disabled": + del file["Path1/user_rois"] + else: + del file["Path1/user_rois/Fsvd"] + del file["Path1/user_rois"].attrs[ + "annotation_enabled" + ] + summary = read_summary(destination) + loaded = summary.paths[0] + self.assertEqual( + loaded.annotation_enabled, state != "disabled" + ) + for source in loaded.sources: + self.assertIsNone(source.df_denoised) + self.assertIsNone(source.events) + for roi in loaded.user_rois: + self.assertIsNone(roi.fsvd) + write_summary(summary, rewritten) + with h5py.File(rewritten, "r") as file: + self.assert_full_schema(file, sources, rois) + for key in ("dF_denoised", "events"): + self.assertTrue(np.isnan( + file[f"Path1/sources/temporal/{key}"][()] + ).all()) + self.assertTrue(np.isnan( + file["Path1/user_rois/Fsvd"][()] + ).all()) + self.assertEqual( + read_summary(rewritten).paths[0].annotation_enabled, + state != "disabled", + ) + + def test_optional_shape_errors_do_not_replace_existing_path(self): + """No broadcasting, transposing or reshaping supplied optional data.""" + for owner, field in ( + ("Source", "df_denoised"), ("Source", "events"), + ("UserRoi", "fsvd"), + ): + for shape in ((7,), (1, 7), (2, 6), (7, 2), (2, 7, 1)): + with self.subTest(owner=owner, field=field, shape=shape): + original = _path() + unrelated = _path(0, 0, False) + unrelated.path_index = 4 + with TemporaryDirectory() as directory: + destination = Path(directory) / "summary.h5" + write_summary(ExperimentSummary( + paths=[original, unrelated], + params={"operator": "original"}, + ), destination) + with h5py.File(destination, "a") as file: + file["Path1"].attrs["marker"] = "retain" + file["params"].attrs["owner"] = "original" + file["unrelated/value"] = np.arange(3) + before = _snapshot(destination) + entries = ( + original.sources if owner == "Source" + else original.user_rois + ) + setattr(entries[1], field, np.ones(shape)) + with self.assertRaisesRegex( + ValueError, rf"{owner}\.{field}\[1\].*shape" + ): + write_summary(ExperimentSummary( + paths=[original], params={"operator": "new"} + ), destination) + after = _snapshot(destination) + self.assertEqual(after.keys(), before.keys()) + for key in before: + np.testing.assert_equal(after[key], before[key]) + + def test_empty_optional_template_shapes_are_validated(self): + """Zero-sized optional tensors must still have the right axes.""" + for key in ("dF_denoised", "events", "Fsvd"): + with self.subTest(key=key), TemporaryDirectory() as directory: + destination = Path(directory) / "summary.h5" + write_summary( + ExperimentSummary(paths=[_path(0, 0)]), destination + ) + original = read_summary(destination).paths[0] + before = _snapshot(destination) + templates = ( + original._empty_roi_arrays if key == "Fsvd" + else original._empty_source_arrays + ) + templates[key] = np.empty((0, 2, 8)) + with self.assertRaisesRegex(ValueError, rf"{key}.*shape"): + write_summary( + ExperimentSummary(paths=[original]), destination + ) + after = _snapshot(destination) + self.assertEqual(after.keys(), before.keys()) + for name in before: + np.testing.assert_equal(after[name], before[name]) + + def test_z_depths_are_always_column_vectors_without_mutation(self): + """Accept vector inputs while persisting the required (fastz, 1) rank.""" + for shape in ((3,), (1, 3), (3, 1)): + with self.subTest(shape=shape), TemporaryDirectory() as directory: + original = _path() + original.z_depths = original.z_depths.reshape(shape) + destination = Path(directory) / "summary.h5" + write_summary(ExperimentSummary(paths=[original]), destination) + with h5py.File(destination, "r") as file: + np.testing.assert_array_equal( + file["Path1/Z_depths"][()], [[2], [4], [8]] + ) + self.assertEqual(file["Path1/Z_depths"].dtype, np.int16) + self.assertEqual(original.z_depths.shape, shape) + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/tests/test_parameter_gui.py b/tests/test_parameter_gui.py index d9881e2..d52c131 100644 --- a/tests/test_parameter_gui.py +++ b/tests/test_parameter_gui.py @@ -72,8 +72,11 @@ def interact(): self.assertEqual(science.analyze_hz, 125) self.assertEqual(science.peakth, 8) self.assertEqual(science.z_tol, 2.5) - self.assertTrue(science.simple_trace_extraction) - self.assertFalse(params.simple_trace_extraction) + for name in ("simple_trace_extraction", "convolution_trace_extraction"): + self.assertFalse(hasattr(science, name)) + self.assertFalse(hasattr(params, name)) + self.assertTrue(science.trace_debias) + self.assertFalse(params.trace_debias) self.assertEqual(params.z_tol, 1.5) self.assertAlmostEqual(science.sparse_fac, math.exp(-2)) labels = [ @@ -97,12 +100,51 @@ def test_cancel_returns_none(self): self.assertIsNone(run_parameter_gui()) self.root.destroy.assert_called_once() - def test_simple_trace_checkbox_preserves_initial_value(self): - """An enabled mode survives accepting the form without edits.""" + def test_removed_trace_flags_are_rejected_before_opening_form(self): + """Removed dictionary keys cannot enter the GUI's base configuration.""" + for name in ("simple_trace_extraction", "convolution_trace_extraction"): + for value in (False, True): + with self.subTest(name=name, value=value): + supplied = {name: value} + with self.assertRaisesRegex(TypeError, name): + run_parameter_gui(supplied) + self.assertEqual(supplied, {name: value}) + self.tk.Tk.assert_not_called() + self.root.mainloop.assert_not_called() + + def test_convolution_controls_preserve_trace_settings(self): + """Accepting the GUI retains the independent trace-only options.""" + params = BandSiloParams( + decay_tau_s=.3, + trace_beta_fraction=.02, trace_max_iter=900, + trace_tolerance=1e-9, trace_debias=True, trace_event_threshold=.001, + ) self.root.mainloop.side_effect = lambda: self.buttons["OK"]() - result = run_parameter_gui(BandSiloParams(simple_trace_extraction=True)) - self.assertTrue(result[0].simple_trace_extraction) + result = run_parameter_gui(params) + self.assertEqual(result[0], params) + self.assertIsNot(result[0], params) + self.assertEqual(result[0].decay_tau_s, .3) + + def test_trace_mode_toggles_are_replaced_by_always_saved_label(self): + """Only debias and ROI checkboxes remain; extraction is not optional.""" + self.root.mainloop.side_effect = lambda: self.buttons["OK"]() + result = run_parameter_gui() + labels = [call.kwargs["text"] for call in self.tk.ttk.Label.call_args_list] + self.assertIn("LS and convolution traces are always saved.", labels) + self.assertEqual(len(self.boolean_variables), 2) + self.assertEqual(self.tk.ttk.Checkbutton.call_count, 2) + self.assertEqual( + [label for label in labels if label.endswith("?")], + ["Debias Convolution Traces?", "Draw User ROIs?"], + ) + for call, variable in zip( + self.tk.ttk.Checkbutton.call_args_list, self.boolean_variables + ): + self.assertIs(call.kwargs["variable"], variable) + for name in ("simple_trace_extraction", "convolution_trace_extraction"): + self.assertFalse(hasattr(result[0], name)) self.assertFalse(result[2].enabled) + self.tk.messagebox.showerror.assert_not_called() def test_invalid_input_shows_error_without_accepting(self): """Invalid scientific values leave the form open for correction.""" diff --git a/tests/test_path_summary.py b/tests/test_path_summary.py index 6f01061..8afa4fb 100644 --- a/tests/test_path_summary.py +++ b/tests/test_path_summary.py @@ -83,6 +83,11 @@ def _result(index, alignment, roi_state="present"): ) / 10 ) + signal[1] = np.nan + denoised = signal + 100 * (index + 1) + section.start + events = np.zeros_like(signal) + events[-1] = np.arange(1, n_sources + 1) + section.start + events[1] = np.nan traces.append( TrialTraceResult( signal, @@ -95,6 +100,8 @@ def _result(index, alignment, roi_state="present"): np.full( (n_frames, n_rois, channels), index + 7.0, dtype=np.float32 ), + denoised, + events, ) ) masks = [np.ones((nz, rows, cols), dtype=bool)] if n_rois else [] @@ -185,6 +192,24 @@ def test_nonzero_two_path_model_and_schema_roundtrip(self): self.assertTrue(all(p.annotation_enabled for p in summary.paths)) for index, path in enumerate(summary.paths): self.assertIsInstance(path.frame_info, FrameInfo) + expected_denoised = np.concatenate([ + r.d_f_denoised for r in results[index].trace_results + ]) + expected_events = np.concatenate([ + r.events for r in results[index].trace_results + ]) + for source_index, source in enumerate(path.sources): + for field, expected_trace in ( + ("df_denoised", expected_denoised), + ("events", expected_events), + ): + values = getattr(source, field) + self.assertEqual(values.shape, (results[index].num_channels, 9)) + np.testing.assert_array_equal(values[0], expected_trace[:, source_index]) + self.assertTrue(np.isnan(values[1:]).all()) + self.assertFalse(np.allclose( + source.df_ls, source.df_denoised, equal_nan=True + )) self.assertIs( path.frame_info.to_dict()["offlineXshifts"], path.frame_info.offlineXshifts, @@ -240,6 +265,11 @@ def test_nonzero_two_path_model_and_schema_roundtrip(self): np.testing.assert_array_equal(actual[key][0], value) for index, path in enumerate(reloaded.paths): self.assertEqual(path.path_index, index) + for before, after in zip(paths[index].sources, path.sources): + for field in ("df_ls", "f0", "df_denoised", "events"): + np.testing.assert_array_equal( + getattr(after, field), getattr(before, field) + ) np.testing.assert_array_equal( path.visualizations.mean_im, results[index].mean_im ) @@ -252,7 +282,17 @@ def test_nonzero_two_path_model_and_schema_roundtrip(self): with h5py.File(model_file, "r") as f: self.assertEqual(f["Path2/user_rois/mask"].shape, (0, 2, 2, 3)) self.assertEqual(f["Path2/user_rois/F"].shape, (0, 1, 9)) + self.assertEqual(f["Path2/user_rois/Fsvd"].shape, (0, 1, 9)) self.assertEqual(f["Path2/user_rois/labels"].shape, (0, 1)) + self.assertTrue(f["Path2/user_rois"].attrs["annotation_enabled"]) + self.assertTrue(np.isnan(f["Path1/user_rois/Fsvd"][()]).all()) + for index, path in enumerate(paths): + for key, field in (("dF_denoised", "df_denoised"), + ("events", "events")): + np.testing.assert_array_equal( + f[f"Path{index + 1}/sources/temporal/{key}"][()], + np.stack([getattr(s, field) for s in path.sources]), + ) for key in FRAME_INFO_KEYS: self.assertEqual(f[f"Path1/frame_info/{key}"].ndim, 2) @@ -290,7 +330,7 @@ def test_explicit_paths_and_typed_frame_info_only(self): self.assertEqual(set(first.frame_info.to_dict()), set(FRAME_INFO_KEYS)) def test_absent_rois_and_overwrite_semantics(self): - """Absent differs from empty; rewrites retain old params.""" + """Disabled differs from enabled-empty by attribute, not group absence.""" data = { "DSframes": [1, 5, 9], **{ @@ -319,8 +359,14 @@ def test_absent_rois_and_overwrite_semantics(self): self.assertTrue(loaded.paths[1].annotation_enabled) self.assertEqual(loaded.params["operator"], "original") with h5py.File(path, "r") as f: - self.assertNotIn("user_rois", f["Path1"]) - self.assertIn("user_rois", f["Path2"]) + rois = f["Path1/user_rois"] + self.assertFalse(rois.attrs["annotation_enabled"]) + self.assertEqual(set(rois), {"mask", "labels", "F", "Fsvd"}) + self.assertEqual(rois["mask"].shape, (0, 1, 3, 4)) + self.assertEqual(rois["labels"].shape, (0, 1)) + self.assertEqual(rois["F"].shape, (0, 2, 9)) + self.assertEqual(rois["Fsvd"].shape, (0, 2, 9)) + self.assertTrue(f["Path2/user_rois"].attrs["annotation_enabled"]) def test_empty_sources_retain_shape_and_dtype(self): """Numerical assembly and codecs retain empty tensor layout.""" From ab5236dbf6987253161a091455220a0c4022644c Mon Sep 17 00:00:00 2001 From: m-xie Date: Tue, 20 Oct 2026 04:34:32 +0000 Subject: [PATCH 48/55] exclude soma from peak detection --- src/giant_python/extraction/band/workflow.py | 8 +++++++- tests/test_band_workflow.py | 10 +++++++++- tests/test_bandsilo_pipeline.py | 6 +++++- 3 files changed, 21 insertions(+), 3 deletions(-) diff --git a/src/giant_python/extraction/band/workflow.py b/src/giant_python/extraction/band/workflow.py index 4fda820..dd0eac8 100644 --- a/src/giant_python/extraction/band/workflow.py +++ b/src/giant_python/extraction/band/workflow.py @@ -324,6 +324,10 @@ def _read(trial_ix): align_hz, params, execution=execution, + exclusion_mask=( + np.logical_or.reduce(soma_masks) + if soma_masks is not None and len(soma_masks) > 0 else None + ), ) # Trace-only options: never passed to localization or its QR solver. @@ -493,6 +497,7 @@ def _localize( params, *, execution=None, + exclusion_mask=None, ): """Compose rho, activity and localization; preserve DoG and RNG order.""" execution = ExecutionOptions() if execution is None else execution @@ -579,7 +584,8 @@ def _localize( ref_d=ref_d, ) source_seeds = get_act_im_peaks( - act_im, peak_th=params.peakth, buffer_size=params.peak_buffer + act_im, peak_th=params.peakth, buffer_size=params.peak_buffer, + exclusion_mask=exclusion_mask, ) act_im_peaks = source_seeds.copy() print(f"Number of detected sources: {act_im_peaks.shape[0]}") diff --git a/tests/test_band_workflow.py b/tests/test_band_workflow.py index 2cdbb4e..74ffba3 100644 --- a/tests/test_band_workflow.py +++ b/tests/test_band_workflow.py @@ -259,7 +259,7 @@ def read_trial( side_effect=lambda data, *args: (np.ones_like(data), 1, 0), ) ) - stack.enter_context( + localize = stack.enter_context( patch.object( wf, "_localize", @@ -341,6 +341,14 @@ def fit_trial(*args, **kwargs): str(table.savedr / "motion_correction" / "p0-selected.h5"), ) self.assertEqual(low_read.call_count, 4) + for dmd_ix, call in enumerate(localize.call_args_list): + mask = call.kwargs["exclusion_mask"] + if rois: + np.testing.assert_array_equal( + mask, roi_selection["user_roi_masks"][f"DMD{dmd_ix + 1}"][0] + ) + else: + self.assertIsNone(mask) self.assertEqual(select_bins.call_count, 2) for call in select_bins.call_args_list: self.assertEqual(call.kwargs["z_tol"], params.z_tol) diff --git a/tests/test_bandsilo_pipeline.py b/tests/test_bandsilo_pipeline.py index b0f9e8d..dcadc58 100644 --- a/tests/test_bandsilo_pipeline.py +++ b/tests/test_bandsilo_pipeline.py @@ -104,6 +104,8 @@ def test_dog_settings_and_all_source_columns_reach_output(self): mot_inds = np.zeros(4, dtype=int) residual = np.ones((2, 4)) rho = np.ones((3, 4)) + exclusion_mask = np.zeros((1, 3, 4), dtype=bool) + exclusion_mask[0, 0, 0] = True seeds = np.array([[0, 1, 2], [0, 2, 2]]) fitted = { "A": torch.ones((12, 2)), @@ -155,7 +157,7 @@ def test_dog_settings_and_all_source_columns_reach_output(self): patch.object(wf.si, "accumulate_activity_image") ) stack.enter_context(patch.object(wf.si, "finalize_activity_image")) - stack.enter_context( + detect = stack.enter_context( patch.object(wf, "get_act_im_peaks", return_value=seeds) ) fit = stack.enter_context( @@ -181,7 +183,9 @@ def test_dog_settings_and_all_source_columns_reach_output(self): 10, BandSiloParams(), execution=ExecutionOptions(verbose=False), + exclusion_mask=exclusion_mask, ) + self.assertIs(detect.call_args.kwargs["exclusion_mask"], exclusion_mask) self.assertEqual(kernels.call_args_list, [call(0.9), call(4.5)]) self.assertEqual( convolution.call_args_list[0].args[1:], ("center", (0, 0)) From ab48fb81cfde21f953d755c917521efd71788071 Mon Sep 17 00:00:00 2001 From: m-xie Date: Thu, 22 Oct 2026 11:04:08 +0000 Subject: [PATCH 49/55] update F0 to be a projection into the background --- .../extraction/band/result_assembly.py | 6 +- src/giant_python/extraction/band/traces.py | 100 ++++++++++++------ src/giant_python/extraction/band/types.py | 7 +- src/giant_python/numerics/baseline.py | 7 +- tests/test_band_workflow.py | 15 ++- tests/test_bandsilo_pipeline.py | 17 +++ tests/test_bandsilo_traces.py | 43 ++++++-- 7 files changed, 146 insertions(+), 49 deletions(-) diff --git a/src/giant_python/extraction/band/result_assembly.py b/src/giant_python/extraction/band/result_assembly.py index c4388e4..d76ed6d 100644 --- a/src/giant_python/extraction/band/result_assembly.py +++ b/src/giant_python/extraction/band/result_assembly.py @@ -82,7 +82,11 @@ def assemble_path_summary(pr: PathResult, path_index: int = 0) -> PathSummary: """ results = pr.trace_results d_f = np.concatenate([r.d_f for r in results], axis=0) - f0_ls = np.concatenate([r.f0_ls for r in results], axis=0) + # F = dF_ls + intermediate F0 + backprojected fit residual. + f0_ls = np.concatenate([ + r.f0_ls if r.f_residual is None else r.f0_ls + r.f_residual + for r in results + ], axis=0) f_full, f0, d_f, _ = assemble_dff( d_f, f0_ls, pr.denoise_window, pr.baseline_window ) diff --git a/src/giant_python/extraction/band/traces.py b/src/giant_python/extraction/band/traces.py index 46e5e1f..e405335 100644 --- a/src/giant_python/extraction/band/traces.py +++ b/src/giant_python/extraction/band/traces.py @@ -6,18 +6,22 @@ onto the trial's downsample grid, bins the frames by motion (matching the low-res motion bins used during NMF), and — per motion bin — projects the fixed spatial profiles ``A_final`` into superpixel space and extracts per-source -temporal traces (``phi``, stored as ``dF_ls``) and background (``F0``) via -least squares or normalized profile-weighted averages. Transient fitting +temporal traces (``phi``, stored as ``dF_ls``) via least squares or +normalized profile-weighted averages. Background (``F0``) is the +backprojection of each projected profile onto the background, and a +backprojection of the fit residual (``data - background - x @ phi``) is kept +for the final F0 estimate. Transient fitting adds separate denoised traces and events, jointly across the trial with fixed -spatial profiles and QR F0. The production workflow always calculates both -least-squares and transient traces. It also computes the +spatial profiles; F0 is never fitted with events. The production workflow +always calculates both least-squares and transient traces. It also computes the global and per-user-ROI fluorescence. The pure compute (:func:`compute_high_res_traces`) is factored out of the IO wrapper (:func:`get_high_res_traces`, which reads the SLAP2 file lazily via :mod:`giant_python.extraction.band.trial_data`) so the numerics are testable without ``slap2_utils``. The workflow owns per-trial scheduling. Results use -``TrialTraceResult``, with denoised traces and events after the original fields. +``TrialTraceResult``, with denoised traces, events and the residual +backprojection after the original fields. """ from __future__ import annotations @@ -71,6 +75,7 @@ def _empty_trace_result( np.full((0, n_soma, num_channels), np.nan, dtype=np.float32), np.full((0, n_sources), np.nan, dtype=np.float32), np.full((0, n_sources), np.nan, dtype=np.float32), + np.full((0, n_sources), np.nan, dtype=np.float32), ) @@ -212,6 +217,18 @@ def _weighted_trace_average( return (weights.T @ data).T +def _backproject(x: torch.Tensor, data: torch.Tensor) -> torch.Tensor: + """Backproject each projected source profile onto a superpixel movie. + + Returns ``x_j^T data / x_j^T x_j`` as (frames, sources): the per-source + coefficient in the same units as the least-squares/denoised ``phi``, but + without de-mixing overlapping sources. Zero profiles return zero, like + the basic least-squares solution; nonfinite samples propagate as NaN. + """ + norm = (x * x).sum(dim=0) + return (data.T @ x) / norm.masked_fill(norm <= 0, 1.0) + + def _solve_trial_phi_f0( data: np.ndarray, background: np.ndarray, @@ -230,12 +247,13 @@ def _solve_trial_phi_f0( analyze_hz: Optional[float] = None, return_transient: bool = False, ) -> tuple: - """Extract per-source traces and background projection. + """Extract per-source traces, background and residual backprojections. For each kept motion bin, shifts ``H`` by the bin's motion, projects - ``A_final`` into superpixel space, and solves for the source temporal - weights (from the background-subtracted residual) and the background - projection ``F0``. + ``A_final`` into superpixel space, solves for the source temporal + weights (from the background-subtracted residual), and backprojects the + profiles onto the background (``F0``) and onto the fit residual + ``data - background - x @ phi``. Parameters ---------- @@ -260,24 +278,30 @@ def _solve_trial_phi_f0( dmd_pixels_per_row : int Grid geometry (column-shift stride). simple_trace_extraction : bool, optional - Use normalized profile-intensity weighted averages instead of least - squares for both dF and F0 (default False). + Use normalized profile-intensity weighted averages for both dF and F0 + instead of least-squares dF and backprojected F0, with a zero + residual term (default False). transient_options : TransientOptions or None - Add benchmark-style constrained dF, keeping separate QR dF and F0. Mutually + Add benchmark-style constrained dF, keeping separate QR dF and + backprojected F0. Mutually exclusive with simple extraction. A failed fit raises RuntimeError. analyze_hz : float or None Nominal grid rate, required for transient fitting; line indices are not seconds. Missing frames must retain their positions in the grid. return_transient : bool - Also return denoised dF and events after phi and f0. Missing-frame - events are NaN; the initial state is not counted as a new event. + Also return denoised dF, events and the residual backprojection after + phi and f0. Missing-frame events are NaN; the initial state is not + counted as a new event. Returns ------- phi, f0 : torch.Tensor of shape (n_frames, n_sources) - Per-source dF and background projection (NaN where no motion bin - applies, where least-squares input frames contain nonfinite samples, - or where weighted averaging has no profile support). + Per-source dF and background backprojection (NaN where no motion bin + applies, where input frames contain nonfinite samples, or where + weighted averaging has no profile support). Outside simple + extraction, F0 is :func:`_backproject` of the background, not a + least-squares fit. The optional residual term is the backprojection + of ``data - background - x @ phi`` (zero for simple extraction). """ if transient_options is not None: if simple_trace_extraction: @@ -293,6 +317,9 @@ def _solve_trial_phi_f0( f0 = torch.full((n_frames, n_sources), float("nan"), dtype=torch.float32) denoised = torch.full_like(phi, float("nan")) events = torch.full_like(phi, float("nan")) + f_residual = torch.full_like( + phi, 0.0 if simple_trace_extraction else float("nan") + ) residual = data - background # One shifted superpixel<-pixel H per trial motion bin (same builder the @@ -313,16 +340,23 @@ def _solve_trial_phi_f0( for motion_idx in mot_inds_to_keep: motion_frames = np.flatnonzero(mot_inds == motion_idx) x = torch.sparse.mm(h_mots[motion_idx], a_final[sel_pix_idxs, :]) - # F0 remains QR for transient fitting (never impose events on F0). - phi[motion_frames, :] = extract( - x, torch.from_numpy(residual[:, motion_frames].astype(np.float32)) + # F0 is never fitted with transient events. + residual_frames = torch.from_numpy( + residual[:, motion_frames].astype(np.float32) ) + phi[motion_frames, :] = extract(x, residual_frames) if transient_options is not None: designs[int(motion_idx)] = x.detach().cpu().numpy() - f0[motion_frames, :] = extract( - x, - torch.from_numpy(background[:, motion_frames].astype(np.float32)), + background_frames = torch.from_numpy( + background[:, motion_frames].astype(np.float32) ) + if simple_trace_extraction: + f0[motion_frames, :] = extract(x, background_frames) + else: + f0[motion_frames, :] = _backproject(x, background_frames) + f_residual[motion_frames, :] = _backproject( + x, residual_frames - x @ phi[motion_frames, :].T + ) if transient_options is not None and n_sources: residual32 = residual.astype(np.float32) @@ -345,7 +379,7 @@ def _solve_trial_phi_f0( events[~valid] = float("nan") if return_transient: - return phi, f0, denoised, events + return phi, f0, denoised, events, f_residual return phi, f0 @@ -488,9 +522,11 @@ def compute_high_res_traces( Maximum absolute rounded z offset from the median, in um (default 1.5). simple_trace_extraction : bool, optional Average superpixels weighted by normalized source profiles instead - of least-squares fitting final dF and F0 traces (default False). + of least-squares fitting dF and backprojecting F0; the residual term + is zero (default False). transient_options : TransientOptions or None - Add constrained dF and events, retaining QR dF/F0 and auxiliary data. + Add constrained dF and events, retaining QR dF, backprojected F0 and + auxiliary data. Fits nonnegative sparse events plus an unpenalized initial state. Public dF retains missing-frame NaNs; nonconvergence raises RuntimeError. analyze_hz : float or None @@ -501,7 +537,8 @@ def compute_high_res_traces( ------- TrialTraceResult Named source traces and auxiliary data. Source trace arrays are - ``(n_frames, n_sources)``; denoised traces/events are separate from d_f. + ``(n_frames, n_sources)``; denoised traces/events are separate from + d_f, and ``f_residual`` is the backprojected fit residual. """ if transient_options is not None: frame_grid = np.asarray(frames) @@ -551,7 +588,7 @@ def compute_high_res_traces( psf2d, ) - phi, f0, denoised, events = _solve_trial_phi_f0( + phi, f0, denoised, events, f_residual = _solve_trial_phi_f0( data, background, unique_motion, @@ -585,6 +622,7 @@ def compute_high_res_traces( f_soma, denoised.numpy(), events.numpy(), + f_residual.numpy(), ) @@ -693,9 +731,11 @@ def get_high_res_traces( Maximum absolute rounded z offset from the median, in um (default 1.5). simple_trace_extraction : bool, optional Average superpixels weighted by normalized source profiles instead - of least-squares fitting final dF and F0 traces (default False). + of least-squares fitting dF and backprojecting F0; the residual term + is zero (default False). transient_options : TransientOptions or None - Add constrained dF/events alongside QR dF/F0 and unchanged profiles. + Add constrained dF/events alongside QR dF, backprojected F0 and + unchanged profiles. The reader's samp_freq supplies the nominal convolution rate. Returns diff --git a/src/giant_python/extraction/band/types.py b/src/giant_python/extraction/band/types.py index fce41d2..0fd5f07 100644 --- a/src/giant_python/extraction/band/types.py +++ b/src/giant_python/extraction/band/types.py @@ -15,8 +15,10 @@ class TrialTraceResult(NamedTuple): with one value per frame. Only output assembly negates reference offsets into saved MATLAB displacement. Skipped trials have zero frames. ``d_f`` is the LS result in production; ``d_f_denoised`` and ``events`` - contain separate convolution outputs. Optional defaults permit legacy - eight-argument construction, but the record now has ten fields. + contain separate convolution outputs. ``f_residual`` is the source + backprojection of ``data - background - x @ d_f``, added with ``f0_ls`` to + ``d_f`` before the final F0 estimate. Optional defaults permit legacy + eight-argument construction, but the record now has eleven fields. """ d_f: np.ndarray @@ -29,6 +31,7 @@ class TrialTraceResult(NamedTuple): user_roi_f: np.ndarray d_f_denoised: np.ndarray | None = None events: np.ndarray | None = None + f_residual: np.ndarray | None = None @dataclass(frozen=True) diff --git a/src/giant_python/numerics/baseline.py b/src/giant_python/numerics/baseline.py index 04ba9d7..67b56b0 100644 --- a/src/giant_python/numerics/baseline.py +++ b/src/giant_python/numerics/baseline.py @@ -166,8 +166,9 @@ def assemble_dff( """Assemble ``F``, re-estimate ``F0``, and form ``dF``/``dFF``. Reconstructs total fluorescence ``F = dF_ls + F0_ls`` from the per-motion - source and background traces (least squares or weighted averages), - re-estimates the baseline with + source traces (least squares or weighted averages) and background terms + (for band extraction, the backprojected background plus the backprojected + fit residual), re-estimates the baseline with :func:`compute_f0`, and computes ``dF = F - F0`` and ``dFF = dF / clip(F0, 1e-4, inf)``. @@ -176,7 +177,7 @@ def assemble_dff( d_f_ls : ndarray of shape (T, n_sources) Concatenated per-source dF traces (``phi``). f0_ls : ndarray of shape (T, n_sources) - Concatenated per-source background traces in the same extraction scale. + Concatenated per-source background terms in the same extraction scale. denoise_window, hull_window : int Windows passed to :func:`compute_f0`. diff --git a/tests/test_band_workflow.py b/tests/test_band_workflow.py index 74ffba3..9fe88ca 100644 --- a/tests/test_band_workflow.py +++ b/tests/test_band_workflow.py @@ -413,7 +413,7 @@ def fit_trial(*args, **kwargs): runtime_result.trace_results[0], TrialTraceResult ) trial = runtime_result.trace_results[0] - self.assertEqual(len(trial), 10) + self.assertEqual(len(trial), 11) fit = fit_results[dmd_ix] np.testing.assert_allclose(trial.d_f_denoised, fit.d_f, rtol=1e-6) np.testing.assert_allclose(trial.events, fit.events, rtol=1e-6) @@ -428,11 +428,16 @@ def fit_trial(*args, **kwargs): expected_ls = np.linalg.lstsq( design, residual[:, selected], rcond=None )[0].T - expected_f0 = np.linalg.lstsq( - design, 3 * residual[:, selected], rcond=None - )[0].T + norm = np.sum(design ** 2, axis=0) + expected_f0 = (3 * residual[:, selected]).T @ design / norm + expected_res = ( + residual[:, selected] - design @ expected_ls.T + ).T @ design / norm np.testing.assert_allclose(trial.d_f[selected], expected_ls, rtol=1e-5) np.testing.assert_allclose(trial.f0_ls[selected], expected_f0, rtol=1e-5) + np.testing.assert_allclose( + trial.f_residual[selected], expected_res, atol=1e-4 + ) self.assertEqual(path.path_index, dmd_ix) np.testing.assert_array_equal(path.z_depths, [[10 + dmd_ix]]) np.testing.assert_array_equal( @@ -463,7 +468,7 @@ def fit_trial(*args, **kwargs): equal_nan=True, ) _, expected_f0, expected_ls, _ = assemble_dff( - trial.d_f, trial.f0_ls, + trial.d_f, trial.f0_ls + trial.f_residual, runtime_result.denoise_window, runtime_result.baseline_window, ) for source in (path.sources[0], saved.sources[0]): diff --git a/tests/test_bandsilo_pipeline.py b/tests/test_bandsilo_pipeline.py index dcadc58..474d3e4 100644 --- a/tests/test_bandsilo_pipeline.py +++ b/tests/test_bandsilo_pipeline.py @@ -518,6 +518,23 @@ def test_two_channels_global_shape(self): self.assertEqual(values.shape, (2, 13)) self.assertTrue(np.isnan(values[1]).all()) + def test_residual_backprojection_enters_final_f0(self): + """F = dF_ls + F0_ls + residual before the final baseline estimate.""" + pr = _path_result() + raw_ls = np.concatenate([r.d_f for r in pr.trace_results]) + raw_f0 = np.concatenate([r.f0_ls for r in pr.trace_results]) + pr.trace_results = [ + r._replace(f_residual=np.full_like(r.d_f, 0.25 * (i + 1))) + for i, r in enumerate(pr.trace_results) + ] + residual = np.concatenate([r.f_residual for r in pr.trace_results]) + _, expected_f0, expected_df, _ = assemble_dff( + raw_ls, raw_f0 + residual, pr.denoise_window, pr.baseline_window + ) + path = pl.assemble_path_summary(pr) + np.testing.assert_array_equal(path.sources[0].f0[0], expected_f0[:, 0]) + np.testing.assert_array_equal(path.sources[0].df_ls[0], expected_df[:, 0]) + def test_independent_traces_stitch_skips_masks_and_legacy_trials(self): """Only LS gets baseline processing; raw fit outputs keep trial order.""" for channels in (1, 2): diff --git a/tests/test_bandsilo_traces.py b/tests/test_bandsilo_traces.py index 2587051..de1553e 100644 --- a/tests/test_bandsilo_traces.py +++ b/tests/test_bandsilo_traces.py @@ -123,11 +123,11 @@ class TestTrialTraceResult(unittest.TestCase): def test_legacy_constructor_defaults_and_extended_fields(self): original = _compute(_trial_inputs()) legacy = TrialTraceResult(*original[:8]) - self.assertEqual(len(legacy), 10) + self.assertEqual(len(legacy), 11) self.assertEqual(legacy._fields, ( "d_f", "f0_ls", "frame_line_idxs", "selected_pixels", "global_f", "reference_offsets", "online_shifts", "user_roi_f", - "d_f_denoised", "events", + "d_f_denoised", "events", "f_residual", )) self.assertIsNone(legacy.d_f_denoised) self.assertIsNone(legacy.events) @@ -272,8 +272,8 @@ def fitted(*args, **kwargs): ) self.assertEqual(len(qr), 2) self.assertEqual(len(pair), 2) - self.assertEqual(len(extended), 4) - self.assertEqual(len(no_fit), 4) + self.assertEqual(len(extended), 5) + self.assertEqual(len(no_fit), 5) self.assertEqual(fit.call_count, 2) for result in (pair, extended, no_fit): for actual, expected in zip(result[:2], qr): @@ -323,6 +323,30 @@ def test_missing_data_and_background_in_motion_bins(self): torch.testing.assert_close(f0[valid_f0], expected_f0[valid_f0]) self.assertTrue(torch.isnan(f0[[3, 7]]).all()) + def test_f0_and_residual_are_profile_backprojections(self): + """F0 is not de-mixed; the residual backprojects data - bg - x phi.""" + profiles = torch.tensor([[1., 2.], [3., 0.], [1., 4.]]) + h_mots = [torch.eye(3).to_sparse()] + rng = np.random.default_rng(0) + data = rng.uniform(5, 10, (3, 6)).astype(np.float32) + background = rng.uniform(1, 3, (3, 6)).astype(np.float32) + args = ( + data, background, np.array([[0, 0]]), np.zeros(6, dtype=int), + np.array([0]), np.empty((0, 2)), np.empty(0), np.arange(3), + profiles, 3, 3, + ) + with patch.object(tr, "build_motion_h_matrices", return_value=h_mots): + phi, f0, _, _, f_res = tr._solve_trial_phi_f0( + *args, return_transient=True + ) + x = profiles.numpy() + norm = np.sum(x ** 2, axis=0) + np.testing.assert_allclose(f0, background.T @ x / norm, rtol=1e-5) + expected_res = (data - background - x @ phi.numpy().T).T @ x / norm + np.testing.assert_allclose(f_res, expected_res, atol=1e-5) + lstsq_f0 = np.linalg.lstsq(x, background, rcond=None)[0].T + self.assertFalse(np.allclose(f0, lstsq_f0, rtol=1e-3)) + class TestBinTrialMotion(unittest.TestCase): """The configured z tolerance controls trial-frame retention.""" @@ -375,7 +399,7 @@ def fit_trial(*args, **kwargs): solve.assert_called_once() self.assertEqual(solve.call_args.args[4], 50) self.assertIs(solve.call_args.kwargs["options"], options) - self.assertEqual(len(out), 10) + self.assertEqual(len(out), 11) for actual, expected in zip(out[:8], baseline[:8]): np.testing.assert_array_equal(actual, expected) np.testing.assert_array_equal( @@ -482,13 +506,15 @@ def test_single_channel_shapes_and_fit(self): self.assertIs(out.user_roi_f, out[7]) self.assertIs(out.d_f_denoised, out[8]) self.assertIs(out.events, out[9]) + self.assertIs(out.f_residual, out[10]) ( phi, f0, frames, sel, global_f, motion, online, f_soma, - denoised, events, + denoised, events, f_residual, ) = out n_sources = inp["a_final"].shape[1] self.assertEqual(phi.shape, (inp["n_frames"], n_sources)) self.assertEqual(f0.shape, (inp["n_frames"], n_sources)) + self.assertEqual(f_residual.shape, phi.shape) for values in (denoised, events): self.assertEqual(values.shape, phi.shape) self.assertTrue(np.isnan(values).all()) @@ -526,7 +552,7 @@ class TestGetHighResTraces(unittest.TestCase): """get_high_res_traces IO wrapper (skip + monkeypatched read).""" def test_skipped_trial_returns_empty(self): - """keep_trial=False returns all ten empty fields without reading.""" + """keep_trial=False returns all eleven empty fields without reading.""" loader = patch.object( tr, "_load_high_res_trial_data", side_effect=AssertionError("read skipped trial") ) @@ -563,10 +589,11 @@ def test_skipped_trial_returns_empty(self): transient_options=TransientOptions(), ) self.assertIsInstance(out, TrialTraceResult) - self.assertEqual(len(out), 10) + self.assertEqual(len(out), 11) self.assertEqual(out[0].shape, (0, inp["a_final"].shape[1])) self.assertEqual(out.d_f_denoised.shape, out.d_f.shape) self.assertEqual(out.events.shape, out.d_f.shape) + self.assertEqual(out.f_residual.shape, out.d_f.shape) self.assertEqual(out[4].shape, (0, 2)) # globalF (0, channels) self.assertEqual(out[2].shape, (0,)) # frames From 1d1b5fb2086bbf4ce7d0487c9e6d9ba633601c86 Mon Sep 17 00:00:00 2001 From: m-xie Date: Thu, 22 Oct 2026 20:43:28 +0000 Subject: [PATCH 50/55] allow setting of random seed --- README.md | 7 +++++ src/giant_python/cli.py | 2 ++ .../extraction/band/localization.py | 29 ++++++++++++++++-- src/giant_python/extraction/band/workflow.py | 2 ++ src/giant_python/models/params.py | 11 +++++++ tests/test_bandsilo_nmf.py | 30 +++++++++++++++++++ tests/test_bandsilo_pipeline.py | 2 +- tests/test_configuration.py | 1 + 8 files changed, 80 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3b410db..400a4f3 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,13 @@ frame z and still satisfy the existing minimum frame count. The value must be positive and finite; the default preserves previous behavior. The CLI and staging example accept `--z-tol`, and the parameter GUI labels it **Z Tolerance (µm)**. +`BandSiloParams(random_seed=0)` seeds the random motion-bin order used by +source localization (NMF and Adam). Each path and outer iteration uses its own +generator derived from the seed, independent of the global torch RNG. The seed +is saved in summary parameters and is available as `--random-seed`. It fixes +the random draws only; floating-point results can still differ across +hardware, thread counts and package versions. + The Python API preceding this cleanup was unreleased and unused. Compatibility-only Python imports and adapters have been removed, not deprecated. MATLAB file-format compatibility is retained separately; there is no production Python API migration requirement. - `extract_band_sources(input, params=None, *, execution=None, annotations=None)` accepts an exact filename or loaded `TrialTable`. Use `Pipeline(..., params=...)` or `Pipeline.from_trial_table(..., params=...).extract()` for a session. diff --git a/src/giant_python/cli.py b/src/giant_python/cli.py index a55d08d..132d496 100644 --- a/src/giant_python/cli.py +++ b/src/giant_python/cli.py @@ -32,6 +32,7 @@ def _add_silo_options(parser: argparse.ArgumentParser) -> None: ("trace_max_iter", int, "Maximum iterations per transient fit stage."), ("trace_tolerance", float, "Transient optimizer tolerance."), ("trace_event_threshold", float, "Absolute event threshold for debias support."), + ("random_seed", int, "Nonnegative seed for the localization motion order."), ): parser.add_argument( "--" + name.replace("_", "-"), type=kind, @@ -137,6 +138,7 @@ def _options_from_args( trace_tolerance=args.trace_tolerance, trace_debias=args.trace_debias, trace_event_threshold=args.trace_event_threshold, + random_seed=args.random_seed, ), ExecutionOptions( max_workers=args.max_workers, diff --git a/src/giant_python/extraction/band/localization.py b/src/giant_python/extraction/band/localization.py index b0100c2..157161f 100644 --- a/src/giant_python/extraction/band/localization.py +++ b/src/giant_python/extraction/band/localization.py @@ -24,7 +24,10 @@ (sorted/pruned) ``source_seeds``. This block is intentionally stochastic (random motion order via -``torch.randperm`` in NMF and Adam); the pipeline does not seed the RNG. The +``torch.randperm`` in NMF and Adam). When :func:`fit_sources` receives a +``seed``, each outer iteration draws from its own ``torch.Generator`` derived +from ``(seed, iteration)``, so a run is reproducible and independent of the +global torch RNG state; with ``seed=None`` the global RNG is used. The ``randperm`` calls are preserved in the same order as the reference, but the unregularized QR temporal solve changes numerical results. The reference's loss-only ``phi`` recomputation and ``print``/warning diagnostics are dropped @@ -367,6 +370,7 @@ def multiplicative_nmf( n_motions: int, sparse_fac: float, max_iters: int, + generator: torch.Generator | None = None, ) -> List[torch.Tensor]: """Motion-binned multiplicative NMF, producing per-motion spatial targets. @@ -396,6 +400,8 @@ def multiplicative_nmf( Sparsification floor subtracted from the max-normalized ``X``. max_iters : int Multiplicative-update iterations per motion bin. + generator : torch.Generator or None + RNG for the motion order; ``None`` uses the global torch RNG. Returns ------- @@ -403,7 +409,7 @@ def multiplicative_nmf( Per-motion fitted superpixel spatial targets ``X_mots``. """ x_mots: List[torch.Tensor | None] = [None] * n_motions - shuffled_indices = torch.randperm(n_motions) + shuffled_indices = torch.randperm(n_motions, generator=generator) for idx in shuffled_indices: # randperm uses an integer dtype, so item() returns a Python int. i = cast(int, idx.item()) @@ -455,6 +461,7 @@ def fit_gaussian_profiles( n_motions: int, learning_rate: float, gd_tol: float, + generator: torch.Generator | None = None, ) -> torch.Tensor: """Adam-fit the Gaussian source parameters to the per-motion NMF targets. @@ -489,6 +496,8 @@ def fit_gaussian_profiles( Base Adam learning rate (scaled per parameter group). gd_tol : float Convergence tolerance on consecutive round losses. + generator : torch.Generator or None + RNG for the motion order; ``None`` uses the global torch RNG. Returns ------- @@ -525,7 +534,7 @@ def fit_gaussian_profiles( a_step = a_step / mass if epoch % n_motions == 0: - shuffled_indices = torch.randperm(n_motions) + shuffled_indices = torch.randperm(n_motions, generator=generator) # Epoch zero initializes the permutation before any index is read. i = cast( int, @@ -703,6 +712,7 @@ def fit_sources( verbose: bool = False, *, temporal_kernel: np.ndarray | None = None, + seed: int | None = None, ) -> dict: """Localize sources by superpixel-space NMF + Gaussian-profile fitting. @@ -748,6 +758,11 @@ def fit_sources( low-resolution temporal weights and SNR all use the filtered data. Raises ValueError if no usable frames remain. If omitted, use the unfiltered residual. + seed : int or None + Nonnegative seed for the random motion order. Each outer iteration + uses a generator seeded from ``(seed, iteration)``, so a change in + the number of Adam epochs in one iteration does not shift the draws + of later iterations. ``None`` uses the global torch RNG. Returns ------- @@ -808,6 +823,12 @@ def fit_sources( desc="Localizing sources (NMF)", verbose=verbose, ): + generator = None + if seed is not None: + iter_seed = np.random.SeedSequence( + [seed, outer_loop_iter] + ).generate_state(1)[0] + generator = torch.Generator().manual_seed(int(iter_seed)) x_mots = multiplicative_nmf( a, h_mots, @@ -818,6 +839,7 @@ def fit_sources( n_motions, sparse_fac, mult_nmf_max_iters, + generator, ) source_params = fit_gaussian_profiles( source_params, @@ -832,6 +854,7 @@ def fit_sources( n_motions, learning_rate, gd_tol, + generator, ) a = project_spatial_profiles( source_params, diff --git a/src/giant_python/extraction/band/workflow.py b/src/giant_python/extraction/band/workflow.py index dd0eac8..98a75e5 100644 --- a/src/giant_python/extraction/band/workflow.py +++ b/src/giant_python/extraction/band/workflow.py @@ -165,6 +165,7 @@ def _params_dict( "trace_tolerance": params.trace_tolerance, "trace_debias": params.trace_debias, "trace_event_threshold": params.trace_event_threshold, + "random_seed": params.random_seed, "draw_user_rois": annotations.enabled, } @@ -602,6 +603,7 @@ def _localize( params.sparse_fac, verbose=execution.verbose, temporal_kernel=temporal_kernel, + seed=params.random_seed, ) return ( act_im, diff --git a/src/giant_python/models/params.py b/src/giant_python/models/params.py index 78d7cb7..c7d467a 100644 --- a/src/giant_python/models/params.py +++ b/src/giant_python/models/params.py @@ -76,6 +76,10 @@ class BandSiloParams: Refit selected events without L1 shrinkage (default False). trace_event_threshold : float Absolute event amplitude threshold for optional debias support. + random_seed : int + Nonnegative seed for the random motion-bin order used by source + localization (NMF and Adam). Each DMD path and outer iteration + draws from its own generator derived from this seed. Notes ----- @@ -103,6 +107,7 @@ class BandSiloParams: trace_tolerance: float = 1e-7 trace_debias: bool = False trace_event_threshold: float = 1e-6 + random_seed: int = 0 def __post_init__(self) -> None: """Validate values supported by the existing numerical path.""" @@ -124,6 +129,12 @@ def __post_init__(self) -> None: raise ValueError("trace_debias must be a bool") for name in ("trace_beta_fraction", "trace_event_threshold"): _nonnegative(name, getattr(self, name)) + if ( + isinstance(self.random_seed, bool) + or not isinstance(self.random_seed, Integral) + or self.random_seed < 0 + ): + raise ValueError("random_seed must be a nonnegative integer") @dataclass diff --git a/tests/test_bandsilo_nmf.py b/tests/test_bandsilo_nmf.py index 7c0deee..dfe9c60 100644 --- a/tests/test_bandsilo_nmf.py +++ b/tests/test_bandsilo_nmf.py @@ -413,6 +413,36 @@ def test_deterministic_under_seed(self): a2 = nmf.fit_sources(seeds.copy(), residual, **kwargs)["A"] np.testing.assert_array_equal(a1.numpy(), a2.numpy()) + def test_explicit_seed_ignores_global_rng(self): + """An explicit seed is reproducible whatever the global torch state.""" + g, seeds, residual, mot_yx, coords, n_pix = self._inputs(seed=1) + kwargs = dict( + h_mots=g["h_mots"], + unique_motion_to_keep_yx=g["umyx"], + mot_inds_yx=mot_yx, + sel_pix_idxs=g["sel_pix_idxs"], + pixel_coords=coords, + n_pixels=n_pix, + d_xy=5, + sparse_fac=float(np.exp(-3.0)), + outer_loop_iters=2, + mult_nmf_max_iters=3, + seed=11, + ) + torch.manual_seed(3) + first = nmf.fit_sources(seeds.copy(), residual, **kwargs) + global_state = torch.get_rng_state() + torch.manual_seed(99) + second = nmf.fit_sources(seeds.copy(), residual, **kwargs) + for key in ("A", "phi_low_res", "source_params", "source_snr"): + np.testing.assert_array_equal( + np.asarray(first[key]), np.asarray(second[key]), err_msg=key + ) + # The seeded path leaves the global RNG untouched. + torch.manual_seed(3) + nmf.fit_sources(seeds.copy(), residual, **kwargs) + self.assertTrue(torch.equal(torch.get_rng_state(), global_state)) + def test_temporal_filter_matches_activity_filter(self): """Reuse activity smoothing and omit its low-coverage edge frames.""" g, seeds, residual, mot_yx, coords, n_pix = self._inputs(seed=1) diff --git a/tests/test_bandsilo_pipeline.py b/tests/test_bandsilo_pipeline.py index 474d3e4..c78dba6 100644 --- a/tests/test_bandsilo_pipeline.py +++ b/tests/test_bandsilo_pipeline.py @@ -306,7 +306,7 @@ def test_persisted_trace_settings_exclude_removed_mode_keys(self): self.assertEqual(metadata["decayTau_s"], params.decay_tau_s) for name in ( "trace_beta_fraction", "trace_max_iter", "trace_tolerance", - "trace_debias", "trace_event_threshold", + "trace_debias", "trace_event_threshold", "random_seed", ): with self.subTest(name=name): self.assertEqual(metadata[name], getattr(params, name)) diff --git a/tests/test_configuration.py b/tests/test_configuration.py index 8642740..e4ec884 100644 --- a/tests/test_configuration.py +++ b/tests/test_configuration.py @@ -109,6 +109,7 @@ def test_trace_tuning_options_use_shared_decay(self): "trace_max_iter": [0, 1.5, True], "trace_beta_fraction": [-1, math.nan, True], "trace_event_threshold": [-1, math.inf], + "random_seed": [-1, 1.5, True, None], }.items(): for value in values: with self.subTest(key=key, value=value), self.assertRaisesRegex(ValueError, key): From 31faeee60947b677a49d4bd2e9a5eb0164aefccb Mon Sep 17 00:00:00 2001 From: m-xie Date: Thu, 22 Oct 2026 21:35:50 +0000 Subject: [PATCH 51/55] update documentation --- CONTRIBUTING.md | 26 +- README.md | 582 +---------- docs/source/architecture.rst | 291 ++---- docs/source/conf.py | 36 + docs/source/file_formats.rst | 930 ++++++++++++++++++ docs/source/index.rst | 155 +-- docs/source/migration.rst | 296 ++---- docs/source/traces.rst | 118 +++ src/giant_python/cli.py | 17 +- src/giant_python/extraction/band/activity.py | 2 +- .../extraction/band/localization.py | 2 +- src/giant_python/extraction/band/operators.py | 4 +- 12 files changed, 1378 insertions(+), 1081 deletions(-) create mode 100644 docs/source/file_formats.rst create mode 100644 docs/source/traces.rst diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3c23d24..a53808c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,5 +1,15 @@ # Contributing +### Development setup + +From the repository root, install the package and development tools with pip 25.1 or later: + +```bash +pip install -e . --group dev +``` + +Alternatively, use `uv sync`. Interactive ROI work also requires tkinter in the Python installation. + ### Linters and testing There are several libraries used to run linters, check documentation, and run tests. @@ -60,15 +70,17 @@ The table below, from [semantic release](https://github.com/semantic-release/sem | `perf(pencil): remove graphiteWidth option`

`BREAKING CHANGE: The graphiteWidth option has been removed.`
`The default graphite width of 10mm is always used for performance reasons.` | ~~Major~~ Breaking Release
(Note that the `BREAKING CHANGE: ` token must be in the footer of the commit and need to be all caps as shown) | ### Documentation -To generate the rst files source files for documentation, run -```bash -sphinx-apidoc -o docs/source/ src -``` -Then to create the documentation HTML files, run + +The maintained Sphinx pages include a curated public API reference; do not regenerate them with `sphinx-apidoc`. +Build the documentation and treat warnings as errors: + ```bash -sphinx-build -b html docs/source/ docs/build/html +sphinx-build -b html -n -W --keep-going docs/source/ docs/build/html ``` -More info on sphinx installation can be found [here](https://www.sphinx-doc.org/en/master/usage/installation.html). + +External API links use Python, NumPy, and h5py inventories downloaded by Sphinx; a clean build requires network access to those documentation sites. + +Keep the README focused on running BandSILo. Put option and staging details in [API and data conventions](docs/source/migration.rst), scientific details in [trace estimation](docs/source/traces.rst), schemas in [file formats](docs/source/file_formats.rst), and ownership decisions in [architecture](docs/source/architecture.rst). ### Read the Docs Deployment Note: Private repositories require **Read the Docs for Business** account. The following instructions are for a public repo. diff --git a/README.md b/README.md index 400a4f3..b7e640a 100644 --- a/README.md +++ b/README.md @@ -1,581 +1,75 @@ -# GIAnT-python +# GIAnT-Python [![License](https://img.shields.io/badge/license-MIT-brightgreen)](LICENSE) -![Code Style](https://img.shields.io/badge/code%20style-black-black) -[![semantic-release: angular](https://img.shields.io/badge/semantic--release-angular-e10079?logo=semantic-release)](https://github.com/semantic-release/semantic-release) ![Python](https://img.shields.io/badge/python->=3.10-blue?logo=python) -An incremental Python port of [GIAnT-MATLAB](https://github.com/AllenNeuralDynamics/GIAnT-MATLAB), currently focused on **SLAP2 BandSILo extraction and annotation from a pre-registered trial table**. This is not a full MATLAB pipeline replacement. +GIAnT-Python is a Python package for processing glutamate imaging data, with advanced capabilities for SLAP2 recordings. -## Implemented scope +Currently, only **SLAP2 Band Scan processing** is implemented: **BandSILo source extraction and Band Scan ROI annotation** from pre-registered recordings. Additional functionality is available in [GIAnT-MATLAB](https://github.com/AllenNeuralDynamics/GIAnT-MATLAB), and efforts are underway to port it into GIAnT-Python. -| Capability | Status | -| --- | --- | -| BandSILo extraction and ROI annotation | Implemented for pre-registered SLAP2 band recordings | -| Trial-table loading | Implemented; accepts an exact filename or a loaded `TrialTable` | -| Generic HDF5 decoding and parameter-group writing | Implemented shared helpers; not a promise of lossless support for every reference field | -| `ExperimentSummary.from_h5()` / `.to_h5()` | Implemented for the emitted band-summary subset, with eager arrays | -| Raw acquisition and alignment reading | Implemented SLAP2 reader access, saved alignment decoding and native TIFF reading | -| Trial-table writing and `AlignmentData` model codecs | Not available; placeholder APIs removed | -| Organization, registration, standard SILo, deconvolution | Not implemented; no placeholder stages or functions are retained | -| Config-file construction, checkpoint/resume | Not implemented; no `from_config()` or `run_all()` API | +## Requirements and scope -The pre-registration inputs must already include the raw recordings and reference TIFFs, alignment files, and the band registration lookup table. Python consumes these externally generated assets; it does not perform registration. See [architecture](docs/source/architecture.rst) and [API and data conventions](docs/source/migration.rst). +Provide an existing trial table, raw recordings with sidecars, reference TIFFs, alignment files, and the band registration lookup table. Python consumes these inputs; it does not organize or register recordings. Standard SILo, standalone deconvolution, trial-table writing, config-file loading, and checkpoint/resume are not supported. -### Placeholder removal — 2026-09-20 +Analysis trials come from the supplied table; Python does not split or merge them. -The earlier requirement to retain future placeholders is superseded: all unimplemented placeholder functions, files and their exports have been removed. This includes the generic numerical stubs, future pipeline stages, `AlignParams`, `AlignmentData`, unsupported IO wrappers and model writers. Working band science, peak fitting, the band-specific NMF solver, native readers, HDF5 writing/codecs and the `Slap2Reader` typing protocol remain. The removal does not alter scientific algorithms. The MATLAB file schemas below remain useful external-format references, not promises of supported Python APIs. +## Installation -## Level of Support - - [ ] Supported: We are releasing this code to the public as a tool we expect others to use. Issues are welcomed, and we expect to address them promptly; pull requests will be vetted by our staff before inclusion. - - [ ] Occasional updates: We are planning on occasional updating this tool with no fixed schedule. Community involvement is encouraged through both issues and pull requests. - - [ ] Unsupported: We are not currently supporting this code, but simply releasing it to the community AS IS but are not able to provide any guarantees of support. The community is welcome to submit issues, but you should not expect an active response. +From the repository root, with Python 3.10 or later: -## Installation -To use the software, in the root directory, run ```bash pip install -e . ``` -To develop the code, run -```bash -pip install -e . --group dev -``` -Note: --group flag is available only in pip versions >=25.1 - -Alternatively, if using `uv`, run -```bash -uv sync -``` +Installation requires Git/network access for the `slap2_utils` dependency. Interactive ROI drawing uses OpenCV and tkinter; tkinter must be available in the Python installation. Headless extraction does not need a display. -Verbose progress uses the declared `tqdm` dependency, imported only when needed. The existing ROI editor uses OpenCV and tkinter (the latter must be available in the Python installation), not napari. Acquisition and GUI imports remain lazy; ordinary model access does not open a GUI. +For development setup and documentation builds, see [CONTRIBUTING.md](CONTRIBUTING.md). -## Supported usage +## Run BandSILo -Supply the **actual** trial-table filename; it need not have a conventional basename. The following function can also receive an already loaded `TrialTable`: +Use the actual trial-table filename, or pass a loaded `TrialTable`: ```python -from giant_python import ( - AnnotationOptions, BandSiloParams, ExecutionOptions, - extract_band_sources, -) - - -def extract_registered_table(table_or_filename): - return extract_band_sources( - table_or_filename, - BandSiloParams(), - execution=ExecutionOptions(max_workers=6, verbose=True), - annotations=AnnotationOptions(enabled=False, interactive=False), - ) -``` - -The returned `ExperimentSummary` is also written beneath the loaded table's `savedr`, in the `source_extraction` subdirectory. Moving or copying the table alone does **not** change `savedr` or `datadr`. For staging and explicit in-memory directory overrides, use [examples/extract_band.py](examples/extract_band.py). +from giant_python import ExecutionOptions, extract_band_sources -For a chainable session, use the same services through `Pipeline`: - -```python -from giant_python import ( - AnnotationOptions, BandSiloParams, ExecutionOptions, Pipeline, +summary = extract_band_sources( + "path/to/trial_table.h5", + execution=ExecutionOptions(max_workers=6, verbose=True), ) - - -def extract_session(table_or_filename): - pipeline = Pipeline.from_trial_table( - table_or_filename, - params=BandSiloParams(), - execution=ExecutionOptions(max_workers=6), - annotations=AnnotationOptions(enabled=False, interactive=False), - ).extract() - return pipeline.summary ``` -`Pipeline` retains the exact source filename when known and accepts a loaded model without serializing/reloading it or guessing a source path from `savedr`. Use `summary.paths` and each `PathSummary.path_index` (zero-based); disk groups are `Path{path_index + 1}`. Arrays are eager, and per-source/ROI entries can share array storage rather than owning independent copies. - -The public workflow methods are `from_trial_table()`, `annotate()` and `extract()`; `extract_sources(...)` is the standalone generic service, not a second `Pipeline` method. `from_config()`, `organize()`, `register()` and `run_all()` are absent, as are the constructor options/attributes `save_dir` and `align_params`. Use the loaded table's `datadr` and `savedr` for explicit directory overrides. Generic `save_struct_h5()` was removed; use the working summary codec for supported band outputs rather than treating the parameter-group writer as a general struct serializer. - -To use existing ROIs without drawing, pass `AnnotationOptions(enabled=True, interactive=False)`. Missing required annotations fail rather than silently omitting ROIs. The explicit `annotate_rois(...)` service or `Pipeline.annotate()` loads existing annotations or draws them when the interactivity policy allows it; annotation files are stored in the table's results directory under `annotations`. There is no implemented config-file loader or resume mode. - -The `giant` CLI exposes only `annotate` and `extract`, each taking an exact trial-table filename. Its only microscope choice is `slap2` and its only scan-mode choice is `band`. Unsupported CLI choices are argument errors; unsupported routing through standalone services raises `ValueError`, not `NotImplementedError`. Pipeline operations delegate to those same services. For headless extraction using existing ROIs, select `--draw-user-rois --headless`. - -### Canonical API and ownership - -`BandSiloParams(z_tol=1.5)` controls the z tolerance in microns for low-resolution -motion-bin selection: bins must lie within this absolute distance of the median -frame z and still satisfy the existing minimum frame count. The value must be -positive and finite; the default preserves previous behavior. The CLI and staging -example accept `--z-tol`, and the parameter GUI labels it **Z Tolerance (µm)**. - -`BandSiloParams(random_seed=0)` seeds the random motion-bin order used by -source localization (NMF and Adam). Each path and outer iteration uses its own -generator derived from the seed, independent of the global torch RNG. The seed -is saved in summary parameters and is available as `--random-seed`. It fixes -the random draws only; floating-point results can still differ across -hardware, thread counts and package versions. - -The Python API preceding this cleanup was unreleased and unused. Compatibility-only Python imports and adapters have been removed, not deprecated. MATLAB file-format compatibility is retained separately; there is no production Python API migration requirement. - -- `extract_band_sources(input, params=None, *, execution=None, annotations=None)` accepts an exact filename or loaded `TrialTable`. Use `Pipeline(..., params=...)` or `Pipeline.from_trial_table(..., params=...).extract()` for a session. -- `resolve_band_options(params=None, execution=None, annotations=None)` returns fresh validated `(BandSiloParams, ExecutionOptions, AnnotationOptions)` objects. Each argument accepts **only its corresponding dataclass, a dictionary of fields belonging to that scope, or `None`**. Mixed dictionaries, cross-scope fields and unknown keys are rejected; they are not redistributed or converted. Caller inputs are not mutated. -- `params` holds science only; channel count is inferred from acquisition metadata and source activity extraction uses channel zero. `execution` holds `max_workers`, `verbose`, and `max_trials`; `annotations` holds `enabled` and `interactive`. Generic services and `Pipeline` take `microscope` / `scan_mode` as explicit routing options; only SLAP2 band extraction is implemented (`scan_mode=None` also selects band). There is no registration configuration class. -- All per-path results are accessed through `ExperimentSummary.paths`. A `PathSummary` owns `sources`, `user_rois`, `visualizations`, `frame_info`, `global_f`, and `z_depths`. There are no flat or aggregate summary properties. Access frame fields as attributes, such as `path.frame_info.offlineXshifts`, or call the shallow `FrameInfo.to_dict()`; `FrameInfo` is not a mapping. -- `giant_python.io.read_summary(path)` returns an `ExperimentSummary`; `giant_python.io.write_summary(summary, path)` accepts that model and returns `None`. Their owner is `giant_python.io.experiment_summary`; the model's `from_h5()` / `to_h5()` delegate there. No public assembled-dictionary writer or alternate summary IO alias is supported. -- `giant_python.extraction.band.result_assembly.assemble_path_summary()` is the band result assembly boundary and returns a `PathSummary`. Scientific backend code belongs in `giant_python.extraction.band`; shared kernels belong in `giant_python.numerics`, not the removed compatibility namespaces. -- `giant_python.gui.parameters_gui.run_parameter_gui(params=None, *, execution=None, annotations=None)` returns `(BandSiloParams, ExecutionOptions, AnnotationOptions)` or `None` on cancellation. It retains the split scopes and does not modify caller-owned options. -- `giant_python.io` exports the working `load_struct_h5`, `read_trial_table`, `read_summary`, `write_summary`, `read_annotations_h5`, `save_annotations_h5`, `load_alignment_data_h5`, `open_slap2_file`, `read_tiff` and `Slap2Reader`. `write_dict_to_h5group` remains a working helper in `giant_python.io.hdf5`, not a universal model serializer. `Slap2Reader` describes the vendor reader contract for typing; it is not an unimplemented workflow stage. - -The shared `gaussian_peaks_integrated()` API in `giant_python.numerics.peaks` accepts only `(N, 5)` parameters `[amp, mu_y, mu_x, sigma_y, sigma_x]`; equal widths must be supplied in both columns. Peak fitting includes an isotropic warm start followed by anisotropic refinement, using canonical five-column parameters with tied widths internally during the first stage. Removing the alternate parameter representation does not retune this fitting policy or the scientific defaults. - -## Epoch and Analysis Trial - -The following describes the reference GIAnT organization concepts. Python currently consumes these structures from an existing trial table rather than implementing the organization stage. - -Epochs are full experimental sessions that can be aligned with each other (i.e. the same field of view and regions of interest are being imaged). Analysis trials are generally contiguous subsets (in time) of an epoch. These analysis trials may not align exactly with experimental trials. - -For SLAP2, analysis trials are the experimental trials if the data was collected using the multi-trial functions of the SLAP2 and each trial is saved off the microscope in a different file. If data was continuously collected on SLAP2, the experiment will be split up into analysis trials of length 200000 lines (~20 sec) to help parallelize processing. - -For data not collected on SLAP2, the reference GIAnT-MATLAB pipeline sets Epochs to be 1 and each file that is selected to be processed is an analysis trial. These analysis trials must be able to be aligned to one another. This is reference behavior, not a supported Python acquisition route. - -## Pipeline Outputs - -### Least-squares and convolution-constrained traces - -Production extraction always computes and saves both standard least-squares -`sources/temporal/dF_ls` and convolution `sources/temporal/dF_denoised`, plus -the fitted `sources/temporal/events`. Each has shape -`(sources, channels, total frames)`. Only activity channel zero is processed; -other channels and missing observations are NaN. Localization/NMF/profile -fitting is unchanged. - -There are no production trace-mode switches: both outputs are mandatory. -The deprecated mode fields and CLI flags have been removed, and are no longer -saved in new summary parameters. Low-level loaded-array functions still support -weighted averages for diagnostics, not production summary extraction. - -The trace-only solver ports the standalone benchmark objective: -`0.5 * ||(data - background) - H_motion A K s||² + beta * sum(s[1:])`, -with nonnegative event amplitudes, a causal unit-peak exponential, and an -unpenalized nonnegative initial state. It solves chronologically over each -complete trial, across motion changes and missing frames; no state crosses -trial boundaries. Returned dF preserves the QR complete-frame validity mask. -F0 is still computed by QR, not by the transient model. Profiles are fixed. - -The trace kernel uses the existing `decay_tau_s` setting (default `0.15` s), -saved as `decayTau_s`, just like localization filtering. Change it with the -existing GUI Decay Tau field or the CLI `--decay-tau-s` flag. There is no -separate trace decay setting. - -Trace-only controls (also available as hyphenated CLI flags): - -| Parameter | Default | Meaning | -| --- | --- | --- | -| `trace_beta_fraction` | `0.01` | Fraction of each trial's positive adjoint scale; zero disables L1 | -| `trace_max_iter` | `500` | Maximum L-BFGS-B iterations per stage | -| `trace_tolerance` | `1e-7` | Objective/gradient tolerance | -| `trace_debias` | `False` | Refit selected events without L1 shrinkage | -| `trace_event_threshold` | `1e-6` | Absolute event threshold for debias support | - -These settings are recorded in summary parameters. A nonconverged fit raises -an error rather than silently saving an unfinished estimate. Increase the -iteration limit or inspect the low-level fit if this occurs. The standard LS -trace and its background retain the existing baseline re-estimation for -`dF_ls` and `F0`. `dF_denoised` is the fitted convolution trace itself, without -a second baseline subtraction. `events` stores nonnegative event amplitudes, -excluding the unpenalized initial state at each trial's first frame; discarded -frames are NaN. Events are not inferred across trial boundaries. - -For loaded-array use, pass `transient_options=TransientOptions(...)` and -`analyze_hz=...` to `compute_high_res_traces`. `TransientOptions` and -`fit_transient_traces` live in `giant_python.extraction.band.transient`. -The latter exposes events, initial state, in-gap model predictions, actual beta, -and convergence diagnostics, and supports an absolute `beta` or explicit causal -kernel. `TrialTraceResult` retains the original eight fields in order and adds -`d_f_denoised` and `events`. Per-trial optimizer diagnostics and latent in-gap -predictions are not added to the summary. - -This is an output-grid estimator: overlapping acquisition-line averaging is -not modeled exactly. In-gap predictions are not observations, and glutamate -events are not automatically neuronal spikes. A smoother trace alone does not -establish improved SNR. Use a fresh output location when comparing modes. - -### Output schema - -The BandSILo writer emits the full experiment-summary layout below: source -`profiles`, `coords`, `dF_ls`, `dF_denoised`, `events`, `F0`, `SNR`; visualizations; -column-vector `Z_depths`; global `F`; frame bookkeeping and shifts; and ROI -`labels`, `mask`, `Fsvd`, `F`. Disabled or empty annotations produce correctly -shaped zero-ROI datasets; the `annotation_enabled` group attribute distinguishes -their policies. ROI `Fsvd` is **not calculated by BandSILo** and is NaN unless -supplied explicitly through the model. It is never copied from raw `F`. - -Older summaries without the new temporal datasets remain readable. Rewriting -missing optional data emits NaNs, not fabricated fits. The entire per-trial -summary remains reference-only and is not emitted. Arbitrary reference files -are not guaranteed to round-trip losslessly through `ExperimentSummary`. - -Trial-table writing is unavailable, and `TrialTable.to_h5` and the `AlignmentData` model have been removed. The shared HDF5 decoder and raw acquisition adapters nevertheless read the inputs needed by BandSILo. Existing summary writes replace supplied path groups but retain unrelated path groups and existing root parameters/layout flags; this is not an atomic replacement or resume mechanism. Prefer a fresh output location for a new analysis. - -### Reading and writing H5 - -The implemented codecs use [`h5py`](https://www.h5py.org/). New Python band summary and annotation files set `row_major` to `1`: the logical dimensions below then match the dataset's h5py `shape`. Reference-only fields retain their intended logical dimensions for future ports; they are not all written by Python. Updating an existing summary preserves its existing layout flag, so use a fresh destination rather than mixing producers in place. - -Files produced by GIAnT-**MATLAB** instead set `row_major` to `0` (or omit it), and their dimension tuples follow MATLAB `size()`. The shared decoder inspects this flag and **reverses multidimensional axes as needed** to recover logical orientation. Schema-specific support is still required; layout conversion does not imply support for every MATLAB file. When in doubt: - -| `row_major` value | Layout | Axis order of the tuples below | -| --- | --- | --- | -| `1` | row-major (h5py / NumPy C order) | matches h5py `shape` as written here | -| `0` or absent | column-major (MATLAB `size()`) | reverse of the tuples here | - -**Emitted vector shapes are preserved on disk.** Band summary frame vectors retain rank two, rather than being squeezed to `(nFrames,)`. For reference-only fields, the listed vector shapes describe the reference schema, not a Python writer guarantee. Layout handling alone does not establish full file interchangeability. - -### Index and coordinate conventions - -These conventions describe the supported subset and the reference fields below: - -- **Line/frame indices** (`first_line`, `last_line`, `DSframes`, `frame_line_idxs`) are **1-indexed** to retain the SLAP2 line-indexing convention. -- **Spatial coordinate fields** in the summaries (`sources/spatial/coords`, `act_im_peaks`, `per_trial_*_coords`, etc.) are **0-indexed** `[z_loc, y_loc, x_loc]`, matching image axis order (`fastz`, rows, cols). `z_loc` is the 0-based index into the `fastz` axis. -- **Annotation coordinate fields** (`position`, `center`) are **0-indexed** `[y_loc, x_loc]` when `coords_zero_indexed = 1`; legacy files without the flag use MATLAB `images.roi` convention (1-indexed `[x, y]`). -- **Images** are stored `rows x cols` (row = Y, col = X). Motion shifts: `motionDSr` / `motionR` are **row** shifts, `motionDSc` / `motionC` are **column** shifts. -- `params/activityChannel` is **1-indexed** into the recording's `numChannels` channels. - -The last field is a reference parameter: do not assume every reference parameter is emitted by the band writer. Python source activity extraction always uses channel zero (zero-based), and source activity arrays for other channels retain NaNs. Channel count is inferred from acquisition metadata, not supplied in `BandSiloParams`. - -**Motion-sign convention:** public `FrameInfo.offlineXshifts`, `offlineYshifts`, and `offlineZshifts` contain alignment displacement with the standard MATLAB SILo sign. Private band reference offsets have the opposite sign and are converted once during `assemble_path_summary()`, not by generic IO. If an experimental file from an unreleased Python checkout exists, establish its producer/alignment provenance before interpreting or changing its shifts; there is no known production Python migration to perform. Never negate MATLAB-produced or already-correct summaries again. Online shifts are unchanged. Matching this sign does not change the existing linear/clamped interpolation to MATLAB's PCHIP/extrapolation. Removing Python compatibility code leaves the HDF5 field spelling `Z_depths`, layout flags, coordinate conversions, scientific defaults, motion-bin ordering, and numerical call order unchanged; no new caching or execution policy is introduced. - -Legend for the trees below (🗄️ file · 📁 group · 🔤 string · 🔢 integer · 📈 numeric · 🖼️ image · ☑️ bool). - -### Trial Table - -Reference input schema: a trial table summarizes file locations and analysis trial structures. Python loads an existing pre-registered table; it does not currently create/save this model or promise to append stage groups. `slap2_info` is SLAP2-specific; the reference pipeline populates `motion_correction` and `source_extraction` downstream. (Model: `giant_python.models.TrialTable` / `Slap2Info`.) - -``` -🗄️ trial_table.h5 - ├ 🔤 datadr - ├ 🔤 savedr - ├ 🔤 filename - ├ 🔢 true_trial_ix - ├ 🔢 epoch - ├ ☑️ row_major - ├ 📁 slap2_info - | ├ 📁 ref_stack - | | └ 📁 Path{1,2} - | | ├ 🖼️ IM - | | ├ 🔢 channels - | | ├ 📈 Zs - | | └ 📈 dmdPixel2SampleTransform - | ├ 🔢 first_line - | ├ 🔢 last_line - | ├ 🔢 trial_start_time_inferred - | └ 🔢 trial_end_time_from_pc - ├ 📁 motion_correction - | ├ 🔤 fn_reg_ds - | ├ 🔤 fn_adata - | ├ 🔤 fn_raw - | ├ ☑️ registration_failed - | ├ 🔢 first_line_original - | └ 📁 align_params - └ 📁 source_extraction - ├ 📁 analysis_params - └ 🔤 fn_raw -``` - -### Alignment Data - -Reference input schema: the external motion correction stage saves an alignment HDF5 file for each trial. Python reads the band fields through `giant_python.io.slap2.load_alignment_data_h5`; there is no `AlignmentData` model or Python registration stage. The full external schema is retained below even where fields are not consumed by the band workflow. - -``` -🗄️ _ALIGNMENTDATA.h5 - ├ ☑️ row_major - ├ 📈 numChannels - ├ 📈 frametime - ├ 📈 alignHz - ├ 📈 motionDSc - ├ 📈 motionDSr - ├ 📈 motionDSz (BandRegistration always; MultiRoiRegistration when refStackTemplate is enabled) - ├ 🖼️ meanIM (StripRegistration and MultiRoiRegistration only; not written by BandRegistration) - ├ 📈 recNegErr (StripRegistration and MultiRoiRegistration only; not written by BandRegistration) - ├ 📈 motionC (StripRegistration / Bergamo only) - ├ 📈 motionR (StripRegistration / Bergamo only) - ├ 📈 motionZ (reserved; not written by any current script) - ├ 📈 brightnessDS (BandRegistration only) - ├ 📈 logLikelihoodDS (BandRegistration only) - ├ 🔢 DSframes (SLAP2 only: MultiRoiRegistration and BandRegistration) - ├ ☑️ registrationFailed (SLAP2 only: MultiRoiRegistration and BandRegistration) - └ 📁 slap2 (SLAP2 only) - ├ 📈 onlineMotionXshift - ├ 📈 onlineMotionYshift - ├ 📈 onlineMotionZshift - ├ 🖼️ varFacDS (MultiRoiRegistration only) - ├ 📈 Z_depths (MultiRoiRegistration only) - ├ 🔢 cropRow (MultiRoiRegistration only) - ├ 🔢 cropCol (MultiRoiRegistration only) - ├ 🖼️ viewC (MultiRoiRegistration only) - ├ 🖼️ viewR (MultiRoiRegistration only) - ├ 🔢 trimRows (MultiRoiRegistration only) - └ 🔢 trimCols (MultiRoiRegistration only) -``` - -### Manual Annotations - -Users can manually annotate pixels to exclude from analysis, or pixels that correspond to soma whose signals should be extracted. When ROIs are annotated, information about the ROIs is saved in the `annotations.h5` file. String fields are stored as UTF-16 code units (`uint16`) for robust MATLAB/Python compatibility. - -``` -🗄️ annotations.h5 - ├ ☑️ row_major - ├ ☑️ coords_zero_indexed - └ 📁 Path{1,2} - ├ 🔤 dr - ├ 🔤 fn - ├ 🔢 n_rois - └ 📁 roi_### - ├ 🔤 type - ├ 🔤 label - ├ 🖼️ mask - ├ 📈 position (polygon only; nVertices x 2 [y_loc, x_loc] when flagged) - ├ 📈 center (circle/ellipse; 1 x 2 [y_loc, x_loc] when flagged) - ├ 📈 semi_axes (ellipse) - ├ 📈 rotation_angle (ellipse) - └ 📈 radius (circle) -``` - -### Experiment Summary - -The final step of the pipeline, source extraction (Source Identification by Activity Localization; SILo), outputs an `experiment_summary.h5` file which contains the extracted sources as well as other useful data about the experiment. Dimensions use one `total frames` axis for all trials from that path stitched in time. (Model: `giant_python.models.ExperimentSummary` / `Source` / `UserRoi` / `Visualizations`.) - -``` -🗄️ experiment_summary.h5 - ├ ☑️ row_major - ├ 📁 params - └ 📁 Path{1,2} - ├ 📈 Z_depths (fastz x 1) - ├ 📁 sources - | ├ 📁 temporal - | | ├ 📈 dF_ls (sources x channels x total frames) - | | ├ 📈 dF_denoised (sources x channels x total frames) - | | ├ 📈 events (sources x channels x total frames) - | | ├ 📈 F0 (sources x channels x total frames) - | | └ 📈 SNR (sources x 1) - | └ 📁 spatial - | ├ 🖼️ profiles (sources x fastz x rows x cols) - | └ 📈 coords (sources x 3 [z_loc, y_loc, x_loc]) - ├ 📁 user_rois - | ├ 🔤 labels (rois x 1) - | ├ 🖼️ mask (rois x fastz x rows x cols) - | ├ 📈 Fsvd (rois x channels x total frames; NaN when unavailable) - | └ 📈 F (rois x channels x total frames) - ├ 📁 visualizations - | ├ 🖼️ mean_im (channels x fastz x rows x cols) - | ├ 🖼️ act_im (fastz x rows x cols) - | └ 🖼️ act_im_peaks (sources x 3 [z_loc, y_loc, x_loc]) - ├ 📁 global - | └ 📈 F (channels x total frames) - └ 📁 frame_info - ├ 📈 offlineXshifts (total frames x 1) - ├ 📈 offlineYshifts (total frames x 1) - ├ 📈 offlineZshifts (total frames x 1) - ├ 📈 onlineXshifts (total frames x 1) - ├ 📈 onlineYshifts (total frames x 1) - ├ 📈 onlineZshifts (total frames x 1) - ├ 🔢 trial_num_frames (trials x 1) - ├ 🔢 frame_line_idxs (total frames x 1) - └ ☑️ discard_frames (total frames x 1) -``` - -**Reference only — not emitted by Python BandSILo.** The reference per-trial summary holds fields that vary across analysis trials. Its trial axis matches the reference trial table (all analysis trials); trials without alignment or source-extraction data are represented as NaN slices. The entire tree below is retained as external-format documentation, not as a Python implementation commitment. - -``` -🗄️ per_trial_summary.h5 - ├ ☑️ row_major - └ 📁 Path{1,2} - ├ 📁 sources - | ├ 📁 temporal - | | └ 📈 per_trial_SNR (trials x sources) - | └ 📁 spatial - | ├ 🖼️ per_trial_profiles (trials x sources x fastz x rows x cols) - | └ 📈 per_trial_coords (trials x sources x 3 [z_loc, y_loc, x_loc]) - └ 📁 visualizations - ├ 🖼️ per_trial_mean_im (trials x channels x fastz x rows x cols) - ├ 🖼️ per_trial_act_im (trials x fastz x rows x cols) - ├ 🖼️ per_trial_act_im_peaks (trials x max_peaks x 3 [z_loc, y_loc, x_loc]) - └ 🔢 per_trial_num_peaks (trials x 1) -``` - -### Band Registration Lookup Table (intermediate file) - -External MATLAB `BandRegistration` builds this reference input under `motion_correction/bandRegLookupTable.h5`. Python BandSILo consumes it; Python registration and lookup-table generation are not implemented. XY search limits (`xPre`, `yPre`, etc.) are shared across paths; per-path superpixel and reference-stack data live under `Path{n}`. - -``` -🗄️ bandRegLookupTable.h5 - ├ 🔢 xPre - ├ 🔢 xPost - ├ 🔢 yPre - ├ 🔢 yPost - ├ ☑️ row_major - └ 📁 Path{1,2} - ├ 📈 likelihood_means (Y x X x Z x C x nSP) - ├ 🔢 allSuperPixelIDs (nSP x 1) - ├ 🔢 sparseMaskInds (N x 2) - ├ 🔢 zPre - ├ 🔢 zPost - └ 📈 fastZ2RefZ -``` - -## File Field Descriptions - -These tables retain the full reference fields for orientation and future ports. They do not override the implemented-subset limits above. In particular, registration outputs are inputs to Python, and all per-trial-summary fields are future-only. - -### `trial_table.h5` - -| Field | Size | Data type | Description | -| --- | --- | --- | --- | -| `row_major` | 1 x 1 | uint8 | Layout flag: `1` = row-major (sizes match h5py `shape`; written by GIAnT-python); `0` = column-major (MATLAB `size()`). **If absent, assume column-major (`0`).** | -| `datadr` | 1 x 1 | string | Data directory location | -| `savedr` | 1 x 1 | string | Results directory location | -| `filename` | nPaths x total trials | string (ragged) | Relative file name from `datadr` | -| `true_trial_ix` | nPaths x total trials | integer | Trial indices unraveled by epochs | -| `epoch` | nPaths x total trials | integer | Epoch numbers | -| `slap2_info` | — | group | Only saved for SLAP2 experiments | -| `slap2_info/ref_stack/Path{1,2}/IM` | image dims | numeric | Reference stack image | -| `slap2_info/ref_stack/Path{1,2}/channels` | 1 x nChannels | numeric | Color channels | -| `slap2_info/ref_stack/Path{1,2}/Zs` | 1 x nZ | numeric | Z positions | -| `slap2_info/ref_stack/Path{1,2}/dmdPixel2SampleTransform` | 3 x 3 | numeric | Transformation matrix | -| `slap2_info/first_line` | nPaths x total trials | integer | First line of each trial (1-indexed) | -| `slap2_info/last_line` | nPaths x total trials | integer | Last line of each trial (1-indexed) | -| `slap2_info/trial_start_time_inferred` | 1 x total trials | integer | Inferred trial start times | -| `slap2_info/trial_end_time_from_pc` | 1 x total trials | integer | Trial end times from PC | -| `motion_correction` | — | group | Written by motion correction stage | -| `motion_correction/fn_reg_ds` | nPaths x total trials | string | Registered + downsampled tif filename | -| `motion_correction/fn_adata` | nPaths x total trials | string | Alignment metadata `_ALIGNMENTDATA.h5` filename | -| `motion_correction/fn_raw` | nPaths x total trials | string | Registered raw-resolution file (Bergamo only) | -| `motion_correction/registration_failed` | nPaths x total trials | bool | Whether registration failed | -| `motion_correction/first_line_original` | nPaths x total trials | integer | Original `slap2_info/first_line` before reVolt adjustment | -| `motion_correction/align_params` | — | group/struct | Alignment parameters used | -| `source_extraction` | — | group | Written by source extraction stage | -| `source_extraction/analysis_params` | — | group/struct | Analysis parameters used | -| `source_extraction/fn_raw` | nPaths x total trials | string | Raw file source extraction reads from per trial | - -### `_ALIGNMENTDATA.h5` - -Top-level fields written by **all three** motion correction backends: `numChannels`, `frametime`, `alignHz`, `motionDSc`, `motionDSr`. `meanIM` and `recNegErr` are written by StripRegistration and MultiRoiRegistration but **not** by BandRegistration. `motionC`/`motionR` are written only by StripRegistration (Bergamo); `DSframes`/`registrationFailed` by both SLAP2 backends (MultiRoiRegistration and BandRegistration); `brightnessDS`/`logLikelihoodDS` by BandRegistration only. The `slap2` group is only populated for SLAP2 experiments. +Annotations are disabled by default. Use `BandSiloParams` for scientific settings, `ExecutionOptions` for workers/progress/trial limits, and `AnnotationOptions` for ROI inclusion and GUI policy. A chainable `Pipeline` is also available; see [API and data conventions](docs/source/migration.rst). -| Field | Size | Data type | Description | -| --- | --- | --- | --- | -| `row_major` | 1 x 1 | uint8 | Layout flag (see above) | -| `numChannels` | 1 x 1 | integer | Number of channels in the recording | -| `meanIM` | channels x rows x cols | single | Per-channel mean of motion-corrected frames (not written by BandRegistration) | -| `frametime` | 1 x 1 | numeric | Seconds per downsampled frame | -| `alignHz` | 1 x 1 | numeric | Frame rate (Hz) at which alignment was performed | -| `motionDSc` | 1 x nDSframes | numeric | Inferred column shift per downsampled frame | -| `motionDSr` | 1 x nDSframes | numeric | Inferred row shift per downsampled frame | -| `motionDSz` | 1 x nDSframes | numeric | Inferred Z shift per downsampled frame; always written by BandRegistration; written by MultiRoiRegistration only when `refStackTemplate` is enabled; never written by StripRegistration | -| `recNegErr` | 1 x nDSframes | numeric | Per-frame reconstruction error; alignment QC metric and used for motion censoring (not written by BandRegistration) | -| `brightnessDS` | nDSframes x channels | numeric | (BandRegistration only) Per-channel brightness/scaling factor at the selected motion shift | -| `logLikelihoodDS` | nDSframes x 1 | numeric | (BandRegistration only) Peak log-likelihood of the motion match per downsampled frame | -| `motionC` | 1 x nFrames | numeric | Column shift upsampled to raw frame rate (Bergamo only) | -| `motionR` | 1 x nFrames | numeric | Row shift upsampled to raw frame rate (Bergamo only) | -| `motionZ` | 1 x nFrames | numeric | (reserved; not written by any current script) Z shift upsampled to raw frame rate | -| `DSframes` | 1 x nDSframes | integer | Line indices of each downsampled frame (SLAP2 only: MultiRoiRegistration and BandRegistration; 1-indexed) | -| `registrationFailed` | 1 x 1 | bool | Whether registration failed for this trial (SLAP2 only: MultiRoiRegistration and BandRegistration) | -| `slap2` | — | group | Only saved for SLAP2 experiments | -| `slap2/varFacDS` | rows x cols x nDSframes | numeric | (MultiRoiRegistration only) Variance factor; multiply pixel intensity to get a value proportional to its variance | -| `slap2/Z_depths` | fastz x 1 | numeric | (MultiRoiRegistration only) Imaged Z depths from microscope metadata | -| `slap2/cropRow` | 1 x 1 | integer | (MultiRoiRegistration only) Row offset to add to ROIs to index into original recording | -| `slap2/cropCol` | 1 x 1 | integer | (MultiRoiRegistration only) Column offset to add to ROIs to index into original recording | -| `slap2/viewC` | (rows+2·maxshift) x (cols+2·maxshift) | numeric | (MultiRoiRegistration only) Column interpolation grid for remapping into saved tiff space | -| `slap2/viewR` | (rows+2·maxshift) x (cols+2·maxshift) | numeric | (MultiRoiRegistration only) Row interpolation grid for remapping into saved tiff space | -| `slap2/trimRows` | 1 x nTrimRows | integer | (MultiRoiRegistration only) Row indices used to remap images from the datafile into saved tiff space | -| `slap2/trimCols` | 1 x nTrimCols | integer | (MultiRoiRegistration only) Column indices used to remap images from the datafile into saved tiff space | -| `slap2/onlineMotionXshift` | 1 x nDSframes | numeric | Online motion-correction X shift from the microscope | -| `slap2/onlineMotionYshift` | 1 x nDSframes | numeric | Online motion-correction Y shift from the microscope | -| `slap2/onlineMotionZshift` | 1 x nDSframes | numeric | Online motion-correction Z shift from the microscope | +**Paths matter:** raw inputs come from the table's `datadr`; results are written under its `savedr`, in `source_extraction/experiment_summary.h5`. Copying the table does not change either directory. For explicit directory overrides and a fresh output location, use [the staging example](examples/extract_band.py). -### `bandRegLookupTable.h5` +### Manual ROIs -`BandRegistration` writes this cached lookup table to `motion_correction/` on the first run and loads it on later runs. XY search limits (`xPre`, `yPre`, etc.) are shared across paths; per-path superpixel and reference-stack data live under `Path{n}` (one group per DMD, in trial-table path order). `Y`, `X`, and `Z` are the row, column, and reference-stack Z dimensions of the motion search cube (`yPre + yPost + 1`, etc.). +- Draw or load annotations with `annotate_rois(table_or_filename)` or `Pipeline.annotate()`. +- Extract using existing ROIs with `annotations=AnnotationOptions(enabled=True, interactive=False)`. Missing required annotations raise an error. +- The CLI provides `giant annotate` and `giant extract`, each taking a trial-table filename. For extraction with existing ROIs and no GUI, select `--draw-user-rois --headless`. -| Field | Size | Data type | Description | -| --- | --- | --- | --- | -| `row_major` | 1 x 1 | uint8 | Layout flag (see above) | -| `xPre` | 1 x 1 | numeric | Maximum column shift searched **before** the reference position (pixels); equals `align_params.maxshiftXY` | -| `xPost` | 1 x 1 | numeric | Maximum column shift searched **after** the reference position (pixels); equals `align_params.maxshiftXY` | -| `yPre` | 1 x 1 | numeric | Maximum row shift searched **before** the reference position (pixels); equals `align_params.maxshiftXY` | -| `yPost` | 1 x 1 | numeric | Maximum row shift searched **after** the reference position (pixels); equals `align_params.maxshiftXY` | -| `Path{n}` | — | group | One group per imaging path (DMD) | -| `Path{n}/likelihood_means` | Y x X x Z x C x nSP | single | Precomputed expected superpixel mean intensity in the padded reference stack at each displacement in the search cube, per channel and superpixel; used as the template for Poisson or correlation motion inference | -| `Path{n}/allSuperPixelIDs` | nSP x 1 | numeric | Unique superpixel keys for this path: `superPixIdx * 100 + zIdx` (band-scan pixels only when `bandOnly` is true) | -| `Path{n}/sparseMaskInds` | N x 2 | numeric | Sparse ROI definition: column 1 = linear DMD pixel index (`rows x cols x numFastZs` layout); column 2 = superpixel index (1 … nSP) | -| `Path{n}/zPre` | 1 x 1 | numeric | Maximum reference-stack Z shift searched **before** the matched plane (planes); capped by `align_params.maxshiftZ` and available reference Z planes | -| `Path{n}/zPost` | 1 x 1 | numeric | Maximum reference-stack Z shift searched **after** the matched plane (planes); capped similarly to `zPre` | -| `Path{n}/fastZ2RefZ` | numFastZs x 1 | numeric | Maps each imaged fast-Z index to the nearest reference-stack Z plane index (used when sampling `likelihood_means`) | +Annotations are stored beneath the table's `savedr` in `annotations/annotations.h5`. -### `annotations.h5` +## Results and cautions -**Indexing conventions.** New files set `coords_zero_indexed` to `1` and store `position` / `center` as **0-indexed `[y_loc, x_loc]`** (row, column). Legacy files without this flag use MATLAB `images.roi` convention: **1-indexed `[x, y]`** (column, row). +The returned `ExperimentSummary` is also saved to disk. Access imaging paths through `summary.paths`; each path contains sources, ROIs, visualizations, frame information, depths, and global fluorescence. Arrays are eager and may share storage. `path_index` is zero-based; disk groups start at `Path1`. -| Field | Size | Data type | Description | -| --- | --- | --- | --- | -| `row_major` | 1 x 1 | uint8 | Layout flag (see above) | -| `coords_zero_indexed` | 1 x 1 | uint8 | When `1`, `position` / `center` are 0-indexed `[y_loc, x_loc]`; when absent or `0`, legacy 1-indexed `[x, y]` | -| `Path{n}` | — | group | One group per imaging path in trial-table order | -| `Path{n}/dr` | 1 x nChars | uint16 | Motion-correction directory used while drawing these ROIs | -| `Path{n}/fn` | 1 x nChars | uint16 | Trial stem used when displaying ROI GUI | -| `Path{n}/n_rois` | 1 x 1 | uint32 | Number of saved ROI entries for this path | -| `Path{n}/roi_###/type` | 1 x nChars | uint16 | ROI geometry type: `polygon`, `circle`, or `ellipse` | -| `Path{n}/roi_###/label` | 1 x nChars | uint16 | User label (e.g., `SOMA`) | -| `Path{n}/roi_###/mask` | rows x cols | uint8 | Binary ROI mask in image coordinates (1 = included pixel) | -| `Path{n}/roi_###/position` | nVertices x 2 | double | Polygon vertices `[y_loc, x_loc]` when `coords_zero_indexed=1`, else legacy `[x, y]` | -| `Path{n}/roi_###/center` | 1 x 2 | double | Circle/ellipse center `[y_loc, x_loc]` when `coords_zero_indexed=1`, else legacy `[x, y]` | -| `Path{n}/roi_###/semi_axes` | 1 x 2 | double | Ellipse semi-axes lengths (ellipse only) | -| `Path{n}/roi_###/rotation_angle` | 1 x 1 | double | Ellipse rotation angle in degrees (ellipse only) | -| `Path{n}/roi_###/radius` | 1 x 1 | double | Circle radius (circle only) | +- Extraction saves least-squares `dF_ls`, convolution-constrained `dF_denoised`, and fitted `events`. Source activity uses channel zero; other channels and missing observations are NaN. Events are not automatically neuronal spikes, and smoother traces do not establish improved SNR. +- ROI `Fsvd` is written as NaN unless supplied explicitly; BandSILo does not compute it. Disabled or empty annotations produce zero-ROI datasets. +- Acquisition line indices are one-based; spatial coordinates are zero-based `[z, y, x]`. Saved offline shifts follow the MATLAB SILo alignment-displacement sign. +- **Use a fresh output location for each analysis.** Updating an existing summary preserves unrelated paths and existing root parameters/layout flags; writes are not atomic replacement or resume. -### `experiment_summary.h5` +## Reference -**Indexing conventions.** Pixel/plane coordinates in `sources/spatial/coords` and related peak/coordinate fields are written **0-indexed** as `[z_loc, y_loc, x_loc]`, matching image axis order (`fastz`, rows, cols). `frame_line_idxs` is kept **1-indexed** to retain the SLAP2 line-indexing convention. +Documentation provenance.[^documentation] -| Field | Size | Data type | Description | -| --- | --- | --- | --- | -| `row_major` | 1 x 1 | uint8 | Layout flag (see above) | -| `params` | — | group/struct | Analysis parameters. `params/activityChannel` is **1-indexed** into the recording's `numChannels` channels — use it to pick the glutamate channel from any `channels x …` dataset (e.g., `global/F`, `sources/temporal/dF_ls`) | -| `Path{n}` | — | group | One group per imaging path | -| `Path{n}/Z_depths` | fastz x 1 | numeric | Z depths per imaging plane (SLAP2 only) | -| `Path{n}/frame_info/offlineXshifts` | total frames x 1 | numeric | Offline registration X shift per frame | -| `Path{n}/frame_info/offlineYshifts` | total frames x 1 | numeric | Offline registration Y shift per frame | -| `Path{n}/frame_info/offlineZshifts` | total frames x 1 | numeric | (optional) Offline registration Z shift per frame; written only when 3D alignment was performed | -| `Path{n}/frame_info/onlineXshifts` | total frames x 1 | numeric | (SLAP2 only) online X shift per frame | -| `Path{n}/frame_info/onlineYshifts` | total frames x 1 | numeric | (SLAP2 only) online Y shift per frame | -| `Path{n}/frame_info/onlineZshifts` | total frames x 1 | numeric | (SLAP2 only) online Z shift per frame | -| `Path{n}/frame_info/trial_num_frames` | trials x 1 | integer | Number of frames contributed by each analysis trial | -| `Path{n}/frame_info/frame_line_idxs` | total frames x 1 | integer | Raw line (SLAP2) or frame (other microscopes) index for each frame in the stitched series. **1-indexed** | -| `Path{n}/frame_info/discard_frames` | total frames x 1 | bool or uint8 | Frame excluded from analysis (e.g., motion censoring) | -| `Path{n}/visualizations/mean_im` | channels x fastz x rows x cols | numeric | Mean registered image per channel / Z slice | -| `Path{n}/visualizations/act_im` | fastz x rows x cols | numeric | Activity / localization summary image | -| `Path{n}/visualizations/act_im_peaks` | sources x 3 | numeric | Activity image peak locations `[z_loc, y_loc, x_loc]`, **0-indexed**; from source row/column coordinates, `z_loc` fixed at `0` | -| `Path{n}/global/F` | channels x total frames | numeric | Fluorescence traces over the whole field (one row per channel) | -| `Path{n}/user_rois/labels` | rois x 1 | string | User-defined ROI labels | -| `Path{n}/user_rois/mask` | rois x fastz x rows x cols | uint8 or bool | Stacked binary masks for each user ROI | -| `Path{n}/user_rois/Fsvd` | rois x channels x total frames | numeric | ROI SVD / projection signals; NaN when unavailable (not computed by BandSILo) | -| `Path{n}/user_rois/F` | rois x channels x total frames | numeric | Raw or baseline-corrected ROI fluorescence | -| `Path{n}/sources/spatial/profiles` | sources x fastz x rows x cols | numeric | Spatial component / pixel weights per source, averaged across trials with footprints | -| `Path{n}/sources/spatial/coords` | sources x 3 | numeric | Source centers per row: `[z_loc, y_loc, x_loc]`, **0-indexed** | -| `Path{n}/sources/temporal/dF_ls` | sources x channels x total frames | numeric | Least-squares ΔF (absolute or scaled) | -| `Path{n}/sources/temporal/dF_denoised` | sources x channels x total frames | numeric | Fitted convolution ΔF, saved alongside least squares | -| `Path{n}/sources/temporal/events` | sources x channels x total frames | numeric | Nonnegative fitted event amplitudes; excludes the trial initial state | -| `Path{n}/sources/temporal/F0` | sources x channels x total frames | numeric | Baseline estimate used for normalization | -| `Path{n}/sources/temporal/SNR` | sources x 1 | numeric | (optional) Signal-to-noise ratio metric; only written when extraction emits per-source SNR | +- [API, configuration, and staging](docs/source/migration.rst) +- [Trace estimation, controls, and scientific caveats](docs/source/traces.rst) +- [File formats, coordinates, and MATLAB interoperability](docs/source/file_formats.rst) +- [Architecture and module ownership](docs/source/architecture.rst) -### `per_trial_summary.h5` +## Support -**All fields in this table are future/reference only; this file is not emitted.** The reference trial axis matches the trial table (all analysis trials); trials without alignment or source-extraction data are left as NaN in the corresponding slices. Coordinate fields use **0-indexed** `[z_loc, y_loc, x_loc]` coordinates. +**Occasional updates:** no fixed update schedule. Community involvement through issues and pull requests is welcome. -| Field | Size | Data type | Description | -| --- | --- | --- | --- | -| `row_major` | 1 x 1 | uint8 | Layout flag (see above) | -| `Path{n}` | — | group | One group per imaging path | -| `Path{n}/visualizations/per_trial_mean_im` | trials x channels x fastz x rows x cols | numeric | Trial-aligned mean registered image per channel / Z slice | -| `Path{n}/visualizations/per_trial_act_im` | trials x fastz x rows x cols | numeric | Trial-aligned activity / localization summary image | -| `Path{n}/visualizations/per_trial_act_im_peaks` | trials x max_peaks x 3 | numeric | Per-trial detected peak locations `[z_loc, y_loc, x_loc]`, **0-indexed**, NaN-padded when a trial has fewer than `max_peaks` | -| `Path{n}/visualizations/per_trial_num_peaks` | trials x 1 | integer | Number of valid peaks per trial | -| `Path{n}/sources/spatial/per_trial_profiles` | trials x sources x fastz x rows x cols | numeric | Spatial component / pixel weights per source per trial | -| `Path{n}/sources/spatial/per_trial_coords` | trials x sources x 3 | numeric | Source centers per trial: `[z_loc, y_loc, x_loc]`, **0-indexed** | -| `Path{n}/sources/temporal/per_trial_SNR` | trials x sources | numeric | Per-source SNR for each analysis trial | +[^documentation]: Current documentation has been generated by large language models (LLMs) from human-verified code. \ No newline at end of file diff --git a/docs/source/architecture.rst b/docs/source/architecture.rst index f85bfd1..5239b07 100644 --- a/docs/source/architecture.rst +++ b/docs/source/architecture.rst @@ -4,197 +4,118 @@ Architecture and ownership Scope ----- -The working path starts with existing registered SLAP2 band inputs and ends -with annotation and BandSILo extraction. Standard SILo, organization, -registration, deconvolution, config loading and checkpoint/resume are not -implemented. As of **2026-09-20**, the earlier requirement to retain future -placeholders is superseded: unimplemented placeholder files, functions and -exports have been removed rather than maintained as future contracts. -MATLAB external-format schemas remain documented separately from Python API -availability; see the repository README and :doc:`migration`. +GIAnT-Python is a package for processing glutamate imaging data, with +advanced capabilities for SLAP2 recordings. The current implementation +supports only SLAP2 Band Scan processing: BandSILo source extraction and +Band Scan ROI annotation from pre-registered inputs. Additional functionality +is available in +`GIAnT-MATLAB `_, +with porting efforts underway. -Directory owners ----------------- +The architecture below describes the implemented functionality. Public usage, +configuration, results and staging are documented in :doc:`migration`; saved +schemas and coordinate conventions are in :doc:`file_formats`; trace +estimation is in :doc:`traces`. -All paths below are relative to ``src/giant_python``. +Module owners +------------- + +All module names below are relative to ``giant_python``. .. list-table:: Canonical responsibilities :header-rows: 1 - :widths: 24 76 + :widths: 44 56 * - Owner - Responsibility - * - ``pipeline/`` - - Public extraction/annotation services, explicit capability routing, - and the chainable ``Pipeline``. Only annotation and extraction are - supported. The CLI and standalone calls use these same services. - * - ``models/`` - - Portable metadata, split configuration and eager public results. - Convenience codecs delegate to shared IO; construction does not - perform band file discovery or motion normalization. - * - ``io/`` - - Shared HDF5 decoding, trial-table reading, emitted experiment-summary - codecs, raw annotation persistence and acquisition adapters. Schema - reading must not silently change the sign of alignment displacement. - * - ``extraction/band/`` - - Working scientific backend: input preparation, geometry, motion bins, - operators, background, activity, localization, summary images, trial - data reduction, traces, ROI geometry and result assembly. ``workflow`` - composes these stages. ``inputs`` owns runtime path resolution, - private motion normalization, lookup/reference/PSF loading and asset - discovery. Pure pixel support and PSF padding/cropping belong to - ``geometry``; numerical assembly belongs to ``result_assembly``. - TIFF decoding stays in ``io.tiff``. - * - ``numerics/`` - - Acquisition-independent baseline, peaks, interpolation and morphology. - Generic registration/deconvolution/NMF/variance stubs were removed; - the working band-specific NMF solver remains in ``localization``. - * - ``gui/`` - - Presentation: the existing ROI editor and parameter form. OpenCV and - tkinter are loaded for interactive work, not model deserialization. - Napari is not the implemented band editor. - * - ``parallel.py`` / ``progress.py`` - - Shared ordered trial mapping and opt-in logging/progress. ``tqdm`` is - imported lazily for verbose progress and is a declared dependency. + * - ``pipeline.extract`` / ``pipeline.annotate`` / ``pipeline.pipeline`` + - Lazy public services, explicit capability routing and the chainable + ``Pipeline``. The CLI and standalone calls use the same services. + * - ``models.params`` / ``models.trial_table`` + - Split configuration and portable input metadata; construction does not + discover band assets or normalize motion. + * - ``models.experiment_summary`` + - Eager ``ExperimentSummary``, ``PathSummary``, ``FrameInfo``, ``Source``, + ``UserRoi`` and ``Visualizations`` models. + * - ``io.experiment_summary`` / ``io.hdf5`` / ``io.trial_table`` + - Summary codecs, shared HDF5 decoding and parameter-group writing, + and trial-table reading. Model convenience codecs delegate here. + * - ``io.slap2`` / ``io.tiff`` + - Raw acquisition/alignment reading and TIFF decoding, without private + band motion-sign normalization. + * - ``extraction.band.inputs`` + - Runtime paths, kept trials, resolved acquisition metadata, private + reference offsets, lookup/reference loading and PSF discovery. + * - ``extraction.band.workflow`` / ``extraction.band.result_assembly`` + - Scientific orchestration and construction of public path models from + numerical results; persistence is delegated to shared IO. + * - ``extraction.band.geometry`` / ``extraction.band.operators`` + - Pure pixel support, PSF padding/cropping and sparse projection operators. + * - ``extraction.band.motion_binning`` / ``extraction.band.background`` / + ``extraction.band.activity`` + - Band motion policies, background estimation and activity calculations. + * - ``extraction.band.localization`` + - Band-specific NMF/localization and spatial profile fitting. + * - ``extraction.band.traces`` / ``extraction.band.transient`` + - Trace extraction and fixed-profile temporal optimization. + * - ``extraction.band.summary_images`` / ``extraction.band.trial_data`` + - Image assembly and superpixel data reduction. + * - ``numerics.baseline`` / ``numerics.peaks`` / + ``numerics.interpolation`` / ``numerics.morphology`` + - Acquisition-independent numerical kernels and helpers. + * - ``io.annotations`` / ``extraction.band.rois`` / + ``extraction.band.annotation`` + - Annotation persistence, pure band ROI geometry and annotation policy. + * - ``gui.roi_editor`` / ``gui.parameters_gui`` + - Interactive presentation with lazy toolkit imports. + * - ``parallel`` / ``progress`` + - Ordered trial mapping and opt-in logging/progress; the declared + ``tqdm`` dependency is imported lazily for verbose progress. * - ``assets/psfs/`` - - Bundled PSF templates; no asset relocation or new cache is required. - -The former Python API was unreleased and unused. Compatibility-only -``bandsilo`` / ``math`` imports and extraction adapters are removed, not -alternative owners or supported aliases. MATLAB file-format support is a -separate IO responsibility. See :doc:`migration` for the canonical owner map. - -Input identity and configuration --------------------------------- - -``extract_band_sources(input, params=None, *, execution=None, annotations=None)`` -accepts an exact filename or a loaded ``TrialTable``. -``Pipeline.from_trial_table`` loads a filename once and retains it as -``trial_table_path``; a loaded table is retained directly. ``source_path`` is -provenance, not a filename reconstructed from ``savedr``. Runtime input -preparation derives raw and alignment locations from the model's explicit -``datadr`` and ``savedr``. Copying the table itself does not relocate them. - -``BandSiloParams`` owns scientific settings, ``ExecutionOptions`` owns worker -count, verbose reporting and the optional trial limit, and -``AnnotationOptions`` owns ROI inclusion and interactivity. -Resolution does not mutate the caller's options to store inferred channel -counts. ``ResolvedAcquisition`` carries resolved backend metadata separately. -Annotation-disabled and enabled-but-empty are distinct states. - -``Pipeline(..., params=...)`` and ``Pipeline.from_trial_table(..., params=...)`` -use the same split options. ``resolve_band_options`` accepts only the matching -dataclass, a dictionary of fields belonging to that scope, or ``None`` for -each argument, and returns fresh validated objects. It does not accept mixed -configuration dictionaries or redistribute routing/run/ROI fields from science. -Routing belongs to explicit ``microscope`` / ``scan_mode`` arguments on the -generic services and ``Pipeline``. Only ``microscope='slap2'`` and band mode -are supported; ``scan_mode=None`` also selects band. Standalone routing checks -raise ``ValueError`` before loading a backend or input, not -``NotImplementedError``. ``AlignParams`` has been removed. - -The public workflow methods on ``Pipeline`` are ``from_trial_table``, -``annotate`` and ``extract``. The constructor accepts ``microscope``, ``params``, -``execution``, ``annotations`` and ``scan_mode``; ``save_dir`` and ``align_params`` -options/attributes no longer exist. Output location comes from ``tt.savedr``. -There are no ``from_config``, ``organize``, ``register`` or ``run_all`` methods, -nor generic stage/builder/corrector classes. The ``giant`` CLI exposes only -``annotate`` and ``extract``, with ``slap2`` / ``band`` as the only routing -choices; invalid commands/options are argument errors. - -``gui.parameters_gui.run_parameter_gui(params=None, *, execution=None, -annotations=None)`` uses these same scopes and returns their validated triple -or ``None`` on cancellation. The existing form, including its natural-log -sparse-factor control, is retained without a combined parameter carrier. - -Typed results and array ownership ---------------------------------- - -``ExperimentSummary.paths`` contains ``PathSummary`` records. Each path owns -its sources, user ROIs, visualizations, depths, global fluorescence and typed -``FrameInfo``. ``path_index`` is **zero-based**; HDF5 names are **one-based**: -``path_index=0`` maps to ``Path1``. Do not infer identity from a path's list -position when reading a subset of saved paths. - -``Source`` and ``UserRoi`` entries can be views of assembled arrays, sharing -data. ``FrameInfo.to_dict()`` is shallow. These are **eager NumPy-backed -results**, not lazy HDF5 proxies, immutable snapshots, or independent copies. -Mutating shared storage can affect other views. Persistence may stack -per-source arrays; the API does not promise zero-copy serialization. - -``ExperimentSummary.paths`` is the only per-path result interface; there are -no flat convenience or aggregate source/ROI properties. ``FrameInfo`` exposes -attributes and ``to_dict()``, not mapping methods or a mapping adapter. - -Inside the backend, ``TrialTraceResult`` names the historical eight-tuple -fields without changing their order. Source traces are frames by sources; -global fluorescence is frames by channels, and ROI traces are frames by ROIs -by channels. Its ``reference_offsets`` are private row/column/Z offsets, -not the public saved displacement. ``PathResult`` collects numerical outputs -before their public summary assembly. - -``result_assembly.assemble_path_summary(PathResult, path_index=0)`` directly -constructs a ``PathSummary``. It is the only band result assembly interface; the former -assembled-dictionary entry point and writer adapters are removed. Private -schema dictionaries inside IO remain implementation details, not a second -public result API. - -Persistence boundaries ----------------------- - -The shared experiment codec supports the **emitted band subset**, not every -field in the README's reference schemas. Future ``dF_denoised``, ``events``, -``Fsvd`` and per-trial summaries are not emitted. Model fields reserved for -those concepts do not establish codec support. Trial-table saving and the -``AlignmentData`` model codecs are unavailable: ``TrialTable.to_h5`` and the -alignment model were removed. Raw alignment reading is implemented separately -by ``io.slap2.load_alignment_data_h5`` and retains saved displacement signs. - -``io.hdf5.load_struct_h5`` and ``write_dict_to_h5group`` are working decoder and -parameter-group writer helpers. Removing the unimplemented ``save_struct_h5`` -does not remove HDF5 writing, annotation persistence or the summary codec. -Native ``io.tiff.read_tiff`` and ``io.slap2.open_slap2_file`` remain; the generic -ScanImage wrappers and ``get_online_motion`` stub do not. ``Slap2Reader`` remains -a structural typing protocol for the vendor reader's existing attributes and -batched ``getLineData`` method; it promises neither a context manager nor a -close method and is not an executable placeholder stage. - -``io.experiment_summary.read_summary(path)`` returns ``ExperimentSummary``; -``write_summary(summary, path)`` accepts that model and returns ``None``. -Both are also exported by ``giant_python.io``. Model convenience methods -delegate to these functions. There are no alternate summary codec aliases or -public assembled-dictionary writers. - -The decoder honors ``row_major`` (absent or zero means MATLAB layout) and -reverses multidimensional axis order when needed. The band writer preserves -rank-two frame vectors and writes new files with ``row_major=1``. Numeric -line indexes remain one-based; output spatial coordinates are zero-based -``[z, y, x]``. Annotation coordinate flags distinguish current zero-based -``[y, x]`` from legacy MATLAB one-based ``[x, y]``. -The HDF5 depth field remains exactly ``Z_depths``; its model attribute is -``PathSummary.z_depths``. These format and coordinate conventions are unaffected -by removal of Python compatibility code. - -Writing an existing summary retains root parameters/layout flags and unrelated -paths, while replacing supplied path groups. This existing behavior is not -transactional output, a fresh-file guarantee, or resume support. Unknown -reference fields are not guaranteed a lossless model round-trip. - -Scientific invariants ---------------------- - -Removing compatibility code and placeholders does not retune defaults, remove -the working band NMF solver, reorder kernels/motion bins/random calls/trials, -change interpolation or scheduling, add caching, or introduce lazy loading. -The public offline-motion contract established by the architecture refactor -matches standard MATLAB SILo: internal reference offsets become alignment -displacement once in ``assemble_path_summary``. IO does not apply another -conversion; online shifts are unchanged. See :doc:`migration` for sign and -provenance cautions, not a production Python migration procedure. - -Shared Gaussian peak evaluation accepts only five-column parameters -``[amp, mu_y, mu_x, sigma_y, sigma_x]``. Equal widths must be explicit in both -columns. Peak fitting retains the implemented isotropic warm start followed -by anisotropic refinement: the first stage ties the two width columns -internally while keeping the canonical five-column representation. Removing -four-column compatibility is not a change to thresholds or fitting policy. \ No newline at end of file + - Bundled PSF templates. + +Backend and IO boundaries +------------------------- + +Runtime input preparation resolves raw and alignment locations from the +loaded table's explicit ``datadr`` and ``savedr``. ``source_path`` records +provenance, not a location reconstructed from ``savedr``. Copying a table +does not relocate its inputs; see :doc:`migration` for staging instructions. +``ResolvedAcquisition`` carries inferred channel metadata separately from +caller-owned scientific options. + +``PathResult`` collects numerical outputs; +``result_assembly.assemble_path_summary(PathResult, path_index=0)`` constructs +the public ``PathSummary``. Schema dictionaries inside IO are implementation +details, not a second public result API. The summary codec supports the +emitted band subset, not lossless round-tripping of every external field. +See :doc:`file_formats` for output coverage and non-atomic write behavior. + +``Slap2Reader`` is a structural typing protocol for the vendor reader's +attributes and batched ``getLineData`` method. It promises neither a context +manager nor a close method. OpenCV and tkinter are loaded for interactive +work, not model deserialization; the implemented ROI editor does not use +napari. + +Numerical boundaries +-------------------- + +Spatial localization/NMF and trace-only temporal optimization are separate +stages. Source activity uses channel zero. See :doc:`traces` for the trace +objective, validity masks, trial boundaries, convergence and reproducibility +controls rather than treating denoising as a selectable extraction mode. + +``numerics.peaks.gaussian_peaks_integrated`` accepts ``(N, 5)`` parameters +``[amp, mu_y, mu_x, sigma_y, sigma_x]``. Both widths must be explicit, even +for isotropic Gaussians. Fitting uses an isotropic warm start followed by +anisotropic refinement: the first stage ties width updates using their +combined derivative while retaining the five-column representation. + +Raw alignment reading preserves saved displacement. Band input preparation +negates it into private reference offsets; ``assemble_path_summary`` converts +those offsets back exactly once for public offline shifts. Generic IO does +not flip signs. Displacement maps reference/view-grid coordinates into raw +sampling coordinates; the inverse mapping uses its negative. No inter-trial +XY correction is invented where the workflow has not computed one. See +:doc:`file_formats` for saved motion signs, producer-provenance cautions and +the linear/clamped versus MATLAB PCHIP/extrapolation distinction. \ No newline at end of file diff --git a/docs/source/conf.py b/docs/source/conf.py index e4ca118..ff5ee46 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -32,10 +32,46 @@ "sphinx.ext.doctest", "sphinx.ext.autodoc", "sphinx.ext.napoleon", + "sphinx.ext.intersphinx", ] templates_path = ["_templates"] exclude_patterns = [] +intersphinx_mapping = { + "python": ("https://docs.python.org/3/", None), + "numpy": ("https://numpy.org/doc/stable/", None), + "h5py": ("https://docs.h5py.org/en/stable/", None), +} + +napoleon_use_param = True +napoleon_use_rtype = True +napoleon_preprocess_types = True +napoleon_type_aliases = { + "Path": "pathlib.Path", + "ndarray": "numpy.ndarray", + "TrialTable": "giant_python.models.trial_table.TrialTable", + "Slap2Info": "giant_python.models.trial_table.Slap2Info", + "BandSiloParams": "giant_python.models.params.BandSiloParams", + "ExecutionOptions": "giant_python.models.params.ExecutionOptions", + "AnnotationOptions": "giant_python.models.params.AnnotationOptions", + "ExperimentSummary": "giant_python.models.experiment_summary.ExperimentSummary", +} + + +def _resolve_public_type_alias(app, env, node, contnode): + """Resolve h5py's runtime class name through its public documentation.""" + if ( + node.get("refdomain") == "py" + and node.get("reftarget") == "h5py._hl.group.Group" + ): + node["reftarget"] = "h5py.Group" + + +def setup(app): + """Normalize private type names before intersphinx resolves references.""" + app.connect("missing-reference", _resolve_public_type_alias, priority=400) + + # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output diff --git a/docs/source/file_formats.rst b/docs/source/file_formats.rst new file mode 100644 index 0000000..347a463 --- /dev/null +++ b/docs/source/file_formats.rst @@ -0,0 +1,930 @@ +File formats +============ + +This reference describes Python BandSILo outputs and the external +`GIAnT-MATLAB `_ +schemas used to interpret pre-registered inputs. The full external field +reference is retained even where Python does not consume or emit a field. +Schema descriptions do not imply that every reference file round-trips +losslessly through ``ExperimentSummary``. + +Python outputs and write behavior +--------------------------------- + +BandSILo emits ``experiment_summary.h5`` with source ``profiles``, ``coords``, +``dF_ls``, ``dF_denoised``, ``events``, ``F0``, and ``SNR``; visualizations; +column-vector ``Z_depths``; global ``F``; frame bookkeeping and shifts; and +ROI ``labels``, ``mask``, ``Fsvd``, and ``F``. See :doc:`traces` for trace +estimation and interpretation. + +Disabled or enabled-but-empty annotations produce correctly shaped zero-ROI +datasets. The ``annotation_enabled`` attribute on each ``Path{n}/user_rois`` +group distinguishes these states. ROI ``Fsvd`` is **not calculated by +BandSILo**: the dataset contains NaN unless values are supplied explicitly +through the model. It is never copied from raw ``F``. Older summaries without +the optional temporal datasets remain readable; rewriting missing optional +data emits NaNs, not fabricated fits. + +``per_trial_summary.h5`` is an **unsupported, external-reference format**; +Python BandSILo does not emit it. Registration products and the band lookup +table are **external inputs**, not Python-generated outputs. Python loads +existing trial tables but does not write them. + +.. warning:: + + Summary writes replace supplied path groups but retain unrelated path + groups and existing root parameters and layout flags. This is neither an + atomic replacement nor a resume mechanism. Prefer a **fresh output + location** for a new analysis, and do not mix producers in place. + +HDF5 layout +----------- + +The implemented codecs use `h5py `_. New Python band +summary and annotation files set ``row_major`` to ``1``: the logical +dimensions below then match each dataset's h5py ``shape``. MATLAB files set +``row_major`` to ``0`` or omit it; their dimension tuples follow MATLAB +``size()``. The shared decoder inspects the flag and reverses multidimensional +axes as needed to recover logical orientation. + +.. list-table:: Layout flag and logical dimensions + :header-rows: 1 + :widths: 18 32 50 + + * - ``row_major`` + - Layout + - Relation to the dimensions listed below + * - ``1`` + - Row-major (h5py / NumPy C order) + - h5py ``shape`` matches the listed logical dimensions. + * - ``0`` or absent + - Column-major (MATLAB ``size()``) + - h5py axes are reversed relative to the listed logical dimensions. + +**Emitted vector shapes are preserved on disk.** Band summary frame vectors +retain rank two rather than being squeezed to ``(nFrames,)``. Reference-only +vector shapes describe the external schema, not a Python writer guarantee. +Layout conversion requires schema-specific interpretation and does not imply +full interchangeability. Updating an existing summary preserves its layout +flag; use a fresh destination when changing producers. + +Indices, coordinates, and motion signs +-------------------------------------- + +* Line/frame indices (``first_line``, ``last_line``, ``DSframes``, and + ``frame_line_idxs``) are **1-indexed**, retaining the SLAP2 convention. +* Summary spatial coordinates (``sources/spatial/coords``, ``act_im_peaks``, + and per-trial coordinate fields) are **0-indexed** + ``[z_loc, y_loc, x_loc]``, matching image axes ``(fastz, rows, cols)``. + ``z_loc`` indexes the fast-z axis; it is not a physical depth. +* Annotation ``position`` and ``center`` use **0-indexed** + ``[y_loc, x_loc]`` when ``coords_zero_indexed=1``. Absent or zero flags + denote legacy MATLAB ``images.roi`` coordinates: **1-indexed** ``[x, y]``. +* Images use ``rows x cols`` (row = Y, column = X). ``motionDSr`` and + ``motionR`` are row shifts; ``motionDSc`` and ``motionC`` are column shifts. +* Python source activity extraction always uses **channel zero**, with NaNs + in other source-activity channels. Channel count comes from acquisition + metadata, not ``BandSiloParams``. The external MATLAB parameter + ``params/activityChannel``, when present, is **1-indexed** into + ``numChannels``; it is not a Python activity-channel selector or a required + emitted parameter. + +.. warning:: + + Public ``FrameInfo.offlineXshifts``, ``offlineYshifts``, and + ``offlineZshifts`` contain alignment displacement with the standard MATLAB + SILo sign. Private band reference offsets have the opposite sign and are + converted once by ``assemble_path_summary()``, not by generic IO. + Establish producer/alignment provenance before interpreting or changing + shifts from experimental files. Never negate MATLAB-produced or + already-correct summaries again. Online shifts are unchanged. Matching + the sign does not make Python's linear/clamped interpolation equivalent + to MATLAB's PCHIP/extrapolation. + +Schema trees +------------ + +Trees show files, groups (names ending in ``/``), and datasets. Dimensions +are logical dimensions under the layout convention above. ``Path{n}`` means +one group per imaging path in trial-table order, with disk numbering starting +at one; ``PathSummary.path_index`` is zero-based. Typical two-path recordings +use ``Path1`` and ``Path2``. + +Trial table: external input +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A trial table summarizes file locations and analysis-trial structure. Python +loads an existing pre-registered table through ``TrialTable`` / ``Slap2Info``; +it does not create/save the model or append reference stage groups. +``slap2_info`` is SLAP2-specific. In the external pipeline, +``motion_correction`` and ``source_extraction`` are populated downstream:: + + trial_table.h5 + ├── datadr + ├── savedr + ├── filename + ├── true_trial_ix + ├── epoch + ├── row_major + ├── slap2_info/ + │ ├── ref_stack/ + │ │ └── Path{n}/ + │ │ ├── IM + │ │ ├── channels + │ │ ├── Zs + │ │ └── dmdPixel2SampleTransform + │ ├── first_line + │ ├── last_line + │ ├── trial_start_time_inferred + │ └── trial_end_time_from_pc + ├── motion_correction/ + │ ├── fn_reg_ds + │ ├── fn_adata + │ ├── fn_raw + │ ├── registration_failed + │ ├── first_line_original + │ └── align_params/ + └── source_extraction/ + ├── analysis_params/ + └── fn_raw + +Alignment data: external registration input +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +External motion correction saves one alignment HDF5 file per trial. Python +reads the band fields through +``giant_python.io.slap2.load_alignment_data_h5``; Python registration is not +implemented. The tree includes external fields not consumed by BandSILo:: + + _ALIGNMENTDATA.h5 + ├── row_major + ├── numChannels + ├── frametime + ├── alignHz + ├── motionDSc + ├── motionDSr + ├── motionDSz (BandRegistration; MultiRoiRegistration with refStackTemplate) + ├── meanIM (StripRegistration and MultiRoiRegistration) + ├── recNegErr (StripRegistration and MultiRoiRegistration) + ├── motionC (StripRegistration / Bergamo only) + ├── motionR (StripRegistration / Bergamo only) + ├── motionZ (reserved; not written by current external scripts) + ├── brightnessDS (BandRegistration only) + ├── logLikelihoodDS (BandRegistration only) + ├── DSframes (MultiRoiRegistration and BandRegistration) + ├── registrationFailed (MultiRoiRegistration and BandRegistration) + └── slap2/ (SLAP2 only) + ├── onlineMotionXshift + ├── onlineMotionYshift + ├── onlineMotionZshift + ├── varFacDS (MultiRoiRegistration only) + ├── Z_depths (MultiRoiRegistration only) + ├── cropRow (MultiRoiRegistration only) + ├── cropCol (MultiRoiRegistration only) + ├── viewC (MultiRoiRegistration only) + ├── viewR (MultiRoiRegistration only) + ├── trimRows (MultiRoiRegistration only) + └── trimCols (MultiRoiRegistration only) + +Manual annotations +~~~~~~~~~~~~~~~~~~ + +Manual annotations identify pixels to exclude from analysis or soma pixels +whose signals should be extracted. ROI information is saved in +``annotations.h5``. String fields use UTF-16 code units (``uint16``) for +MATLAB/Python compatibility:: + + annotations.h5 + ├── row_major + ├── coords_zero_indexed + └── Path{n}/ + ├── dr + ├── fn + ├── n_rois + └── roi_###/ + ├── type + ├── label + ├── mask + ├── position (polygon; nVertices x 2 [y_loc, x_loc] when flagged) + ├── center (circle/ellipse; 1 x 2 [y_loc, x_loc] when flagged) + ├── semi_axes (ellipse) + ├── rotation_angle (ellipse) + └── radius (circle) + +Experiment summary +~~~~~~~~~~~~~~~~~~ + +Source extraction (Source Identification by Activity Localization; SILo) +writes ``experiment_summary.h5`` with extracted sources and experiment data. +The ``total frames`` axis stitches trials from that path in time. The models +are ``ExperimentSummary``, ``Source``, ``UserRoi``, and ``Visualizations``:: + + experiment_summary.h5 + ├── row_major + ├── params/ + └── Path{n}/ + ├── Z_depths (fastz x 1) + ├── sources/ + │ ├── temporal/ + │ │ ├── dF_ls (sources x channels x total frames) + │ │ ├── dF_denoised (sources x channels x total frames) + │ │ ├── events (sources x channels x total frames) + │ │ ├── F0 (sources x channels x total frames) + │ │ └── SNR (sources x 1) + │ └── spatial/ + │ ├── profiles (sources x fastz x rows x cols) + │ └── coords (sources x 3 [z_loc, y_loc, x_loc]) + ├── user_rois/ [attribute: annotation_enabled] + │ ├── labels (rois x 1) + │ ├── mask (rois x fastz x rows x cols) + │ ├── Fsvd (rois x channels x total frames; NaN unless supplied) + │ └── F (rois x channels x total frames) + ├── visualizations/ + │ ├── mean_im (channels x fastz x rows x cols) + │ ├── act_im (fastz x rows x cols) + │ └── act_im_peaks (sources x 3 [z_loc, y_loc, x_loc]) + ├── global/ + │ └── F (channels x total frames) + └── frame_info/ + ├── offlineXshifts (total frames x 1) + ├── offlineYshifts (total frames x 1) + ├── offlineZshifts (total frames x 1) + ├── onlineXshifts (total frames x 1) + ├── onlineYshifts (total frames x 1) + ├── onlineZshifts (total frames x 1) + ├── trial_num_frames (trials x 1) + ├── frame_line_idxs (total frames x 1) + └── discard_frames (total frames x 1) + +Per-trial summary: unsupported external reference +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Python BandSILo does not emit this file.** The external format holds fields +that vary across analysis trials. Its trial axis matches the reference trial +table (all analysis trials); trials without alignment or source-extraction +data are represented by NaN slices:: + + per_trial_summary.h5 + ├── row_major + └── Path{n}/ + ├── sources/ + │ ├── temporal/ + │ │ └── per_trial_SNR (trials x sources) + │ └── spatial/ + │ ├── per_trial_profiles (trials x sources x fastz x rows x cols) + │ └── per_trial_coords (trials x sources x 3 [z_loc, y_loc, x_loc]) + └── visualizations/ + ├── per_trial_mean_im (trials x channels x fastz x rows x cols) + ├── per_trial_act_im (trials x fastz x rows x cols) + ├── per_trial_act_im_peaks (trials x max_peaks x 3 [z_loc, y_loc, x_loc]) + └── per_trial_num_peaks (trials x 1) + +Band registration lookup table: external input +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +External MATLAB ``BandRegistration`` builds +``motion_correction/bandRegLookupTable.h5``. Python consumes this file; +registration and lookup-table generation are not implemented. XY search +limits are shared across paths; superpixel and reference-stack data are +per-path:: + + bandRegLookupTable.h5 + ├── xPre + ├── xPost + ├── yPre + ├── yPost + ├── row_major + └── Path{n}/ + ├── likelihood_means (Y x X x Z x C x nSP) + ├── allSuperPixelIDs (nSP x 1) + ├── sparseMaskInds (N x 2) + ├── zPre + ├── zPost + └── fastZ2RefZ + +Field descriptions +------------------ + +These tables retain the full external reference fields alongside Python +outputs. Registration outputs are inputs to Python; all per-trial-summary +fields are external-reference only. The layout flag has the same meaning in +each table: ``1`` means the listed dimensions match h5py ``shape``; ``0`` or +absence means MATLAB layout. + +Trial table fields +~~~~~~~~~~~~~~~~~~ + +.. list-table:: ``trial_table.h5`` (external input) + :header-rows: 1 + :widths: 30 20 12 38 + + * - Field + - Size + - Data type + - Description + * - ``row_major`` + - 1 x 1 + - uint8 + - Layout flag; if absent, assume column-major (``0``). + * - ``datadr`` + - 1 x 1 + - string + - Data directory location. + * - ``savedr`` + - 1 x 1 + - string + - Results directory location. + * - ``filename`` + - nPaths x total trials + - string (ragged) + - Relative file name from ``datadr``. + * - ``true_trial_ix`` + - nPaths x total trials + - integer + - Trial indices unraveled by epochs. + * - ``epoch`` + - nPaths x total trials + - integer + - Epoch numbers. + * - ``slap2_info`` + - — + - group + - Only saved for SLAP2 experiments. + * - ``slap2_info/ref_stack/Path{n}/IM`` + - image dims + - numeric + - Reference stack image. + * - ``slap2_info/ref_stack/Path{n}/channels`` + - 1 x nChannels + - numeric + - Color channels. + * - ``slap2_info/ref_stack/Path{n}/Zs`` + - 1 x nZ + - numeric + - Z positions. + * - ``slap2_info/ref_stack/Path{n}/dmdPixel2SampleTransform`` + - 3 x 3 + - numeric + - Transformation matrix. + * - ``slap2_info/first_line`` + - nPaths x total trials + - integer + - First line of each trial (1-indexed). + * - ``slap2_info/last_line`` + - nPaths x total trials + - integer + - Last line of each trial (1-indexed). + * - ``slap2_info/trial_start_time_inferred`` + - 1 x total trials + - integer + - Inferred trial start times. + * - ``slap2_info/trial_end_time_from_pc`` + - 1 x total trials + - integer + - Trial end times from PC. + * - ``motion_correction`` + - — + - group + - Written by the external motion-correction stage. + * - ``motion_correction/fn_reg_ds`` + - nPaths x total trials + - string + - Registered and downsampled TIFF filename. + * - ``motion_correction/fn_adata`` + - nPaths x total trials + - string + - Alignment metadata ``_ALIGNMENTDATA.h5`` filename. + * - ``motion_correction/fn_raw`` + - nPaths x total trials + - string + - Registered raw-resolution file (Bergamo only). + * - ``motion_correction/registration_failed`` + - nPaths x total trials + - bool + - Whether registration failed. + * - ``motion_correction/first_line_original`` + - nPaths x total trials + - integer + - Original ``slap2_info/first_line`` before reVolt adjustment. + * - ``motion_correction/align_params`` + - — + - group/struct + - Alignment parameters used. + * - ``source_extraction`` + - — + - group + - Written by the external source-extraction stage. + * - ``source_extraction/analysis_params`` + - — + - group/struct + - Analysis parameters used. + * - ``source_extraction/fn_raw`` + - nPaths x total trials + - string + - Raw file source extraction reads from per trial. + +Alignment data fields +~~~~~~~~~~~~~~~~~~~~~ + +All three external motion-correction backends write ``numChannels``, +``frametime``, ``alignHz``, ``motionDSc``, and ``motionDSr``. +``meanIM`` and ``recNegErr`` are written by StripRegistration and +MultiRoiRegistration, but not BandRegistration. ``motionC`` and ``motionR`` +are StripRegistration/Bergamo-only. Both SLAP2 backends write ``DSframes`` +and ``registrationFailed``. ``brightnessDS`` and ``logLikelihoodDS`` are +BandRegistration-only. The ``slap2`` group is populated only for SLAP2. + +.. list-table:: ``_ALIGNMENTDATA.h5`` (external registration input) + :header-rows: 1 + :widths: 28 22 12 38 + + * - Field + - Size + - Data type + - Description + * - ``row_major`` + - 1 x 1 + - uint8 + - Layout flag. + * - ``numChannels`` + - 1 x 1 + - integer + - Number of channels in the recording. + * - ``meanIM`` + - channels x rows x cols + - single + - Per-channel mean of motion-corrected frames; not written by BandRegistration. + * - ``frametime`` + - 1 x 1 + - numeric + - Seconds per downsampled frame. + * - ``alignHz`` + - 1 x 1 + - numeric + - Frame rate (Hz) at which alignment was performed. + * - ``motionDSc`` + - 1 x nDSframes + - numeric + - Inferred column shift per downsampled frame. + * - ``motionDSr`` + - 1 x nDSframes + - numeric + - Inferred row shift per downsampled frame. + * - ``motionDSz`` + - 1 x nDSframes + - numeric + - Inferred Z shift per downsampled frame. Always written by BandRegistration; + written by MultiRoiRegistration only with ``refStackTemplate`` enabled; + never written by StripRegistration. + * - ``recNegErr`` + - 1 x nDSframes + - numeric + - Per-frame reconstruction error; alignment QC metric used for motion + censoring. Not written by BandRegistration. + * - ``brightnessDS`` + - nDSframes x channels + - numeric + - BandRegistration-only per-channel brightness/scaling factor at the + selected motion shift. + * - ``logLikelihoodDS`` + - nDSframes x 1 + - numeric + - BandRegistration-only peak log-likelihood of the motion match per + downsampled frame. + * - ``motionC`` + - 1 x nFrames + - numeric + - Column shift upsampled to raw frame rate (Bergamo only). + * - ``motionR`` + - 1 x nFrames + - numeric + - Row shift upsampled to raw frame rate (Bergamo only). + * - ``motionZ`` + - 1 x nFrames + - numeric + - Reserved Z shift upsampled to raw frame rate; not written by any + current external script. + * - ``DSframes`` + - 1 x nDSframes + - integer + - Line indices of each downsampled frame (1-indexed); SLAP2 only, + MultiRoiRegistration and BandRegistration. + * - ``registrationFailed`` + - 1 x 1 + - bool + - Whether registration failed for this trial; SLAP2 only, + MultiRoiRegistration and BandRegistration. + * - ``slap2`` + - — + - group + - Only saved for SLAP2 experiments. + * - ``slap2/varFacDS`` + - rows x cols x nDSframes + - numeric + - MultiRoiRegistration-only variance factor; multiply pixel intensity + by it to obtain a value proportional to its variance. + * - ``slap2/Z_depths`` + - fastz x 1 + - numeric + - MultiRoiRegistration-only imaged Z depths from microscope metadata. + * - ``slap2/cropRow`` + - 1 x 1 + - integer + - MultiRoiRegistration-only row offset to add to ROIs to index the + original recording. + * - ``slap2/cropCol`` + - 1 x 1 + - integer + - MultiRoiRegistration-only column offset to add to ROIs to index the + original recording. + * - ``slap2/viewC`` + - (rows+2·maxshift) x (cols+2·maxshift) + - numeric + - MultiRoiRegistration-only column interpolation grid for remapping + into saved TIFF space. + * - ``slap2/viewR`` + - (rows+2·maxshift) x (cols+2·maxshift) + - numeric + - MultiRoiRegistration-only row interpolation grid for remapping + into saved TIFF space. + * - ``slap2/trimRows`` + - 1 x nTrimRows + - integer + - MultiRoiRegistration-only row indices used to remap images from + the datafile into saved TIFF space. + * - ``slap2/trimCols`` + - 1 x nTrimCols + - integer + - MultiRoiRegistration-only column indices used to remap images from + the datafile into saved TIFF space. + * - ``slap2/onlineMotionXshift`` + - 1 x nDSframes + - numeric + - Online motion-correction X shift from the microscope. + * - ``slap2/onlineMotionYshift`` + - 1 x nDSframes + - numeric + - Online motion-correction Y shift from the microscope. + * - ``slap2/onlineMotionZshift`` + - 1 x nDSframes + - numeric + - Online motion-correction Z shift from the microscope. + +Band registration lookup fields +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +External ``BandRegistration`` writes the cached lookup table into +``motion_correction/`` on its first run and loads it on later runs. XY search +limits (``xPre``, ``yPre``, etc.) are shared across paths; each ``Path{n}`` +holds superpixel and reference-stack data for one DMD in trial-table order. +``Y``, ``X``, and ``Z`` denote row, column, and reference-stack Z dimensions +of the search cube (``yPre + yPost + 1``, etc.). + +.. list-table:: ``bandRegLookupTable.h5`` (external registration input) + :header-rows: 1 + :widths: 28 22 12 38 + + * - Field + - Size + - Data type + - Description + * - ``row_major`` + - 1 x 1 + - uint8 + - Layout flag. + * - ``xPre`` + - 1 x 1 + - numeric + - Maximum column shift searched before the reference position (pixels); + equals ``align_params.maxshiftXY``. + * - ``xPost`` + - 1 x 1 + - numeric + - Maximum column shift searched after the reference position (pixels); + equals ``align_params.maxshiftXY``. + * - ``yPre`` + - 1 x 1 + - numeric + - Maximum row shift searched before the reference position (pixels); + equals ``align_params.maxshiftXY``. + * - ``yPost`` + - 1 x 1 + - numeric + - Maximum row shift searched after the reference position (pixels); + equals ``align_params.maxshiftXY``. + * - ``Path{n}`` + - — + - group + - One group per imaging path (DMD). + * - ``Path{n}/likelihood_means`` + - Y x X x Z x C x nSP + - single + - Precomputed expected superpixel mean intensity in the padded reference + stack at each search-cube displacement, per channel and superpixel; + template for Poisson or correlation motion inference. + * - ``Path{n}/allSuperPixelIDs`` + - nSP x 1 + - numeric + - Unique keys ``superPixIdx * 100 + zIdx`` for this path; band-scan pixels + only when ``bandOnly`` is true. + * - ``Path{n}/sparseMaskInds`` + - N x 2 + - numeric + - Sparse ROI definition. Column 1: linear DMD pixel index in + ``rows x cols x numFastZs`` layout. Column 2: superpixel index (1 … nSP). + * - ``Path{n}/zPre`` + - 1 x 1 + - numeric + - Maximum reference-stack Z shift searched before the matched plane + (planes); capped by ``align_params.maxshiftZ`` and available reference + Z planes. + * - ``Path{n}/zPost`` + - 1 x 1 + - numeric + - Maximum reference-stack Z shift searched after the matched plane + (planes); capped as for ``zPre``. + * - ``Path{n}/fastZ2RefZ`` + - numFastZs x 1 + - numeric + - Maps each imaged fast-Z index to its nearest reference-stack Z plane + index, used when sampling ``likelihood_means``. + +Annotation fields +~~~~~~~~~~~~~~~~~ + +New files set ``coords_zero_indexed=1`` and store ``position`` / ``center`` +as **0-indexed** ``[y_loc, x_loc]`` (row, column). Legacy files without the +flag use MATLAB ``images.roi`` convention: **1-indexed** ``[x, y]`` +(column, row). + +.. list-table:: ``annotations.h5`` + :header-rows: 1 + :widths: 28 22 12 38 + + * - Field + - Size + - Data type + - Description + * - ``row_major`` + - 1 x 1 + - uint8 + - Layout flag. + * - ``coords_zero_indexed`` + - 1 x 1 + - uint8 + - When ``1``, ``position`` / ``center`` are 0-indexed + ``[y_loc, x_loc]``; when absent or ``0``, legacy 1-indexed ``[x, y]``. + * - ``Path{n}`` + - — + - group + - One group per imaging path in trial-table order. + * - ``Path{n}/dr`` + - 1 x nChars + - uint16 + - Motion-correction directory used while drawing these ROIs. + * - ``Path{n}/fn`` + - 1 x nChars + - uint16 + - Trial stem used when displaying the ROI GUI. + * - ``Path{n}/n_rois`` + - 1 x 1 + - uint32 + - Number of saved ROI entries for this path. + * - ``Path{n}/roi_###/type`` + - 1 x nChars + - uint16 + - ROI geometry: ``polygon``, ``circle``, or ``ellipse``. + * - ``Path{n}/roi_###/label`` + - 1 x nChars + - uint16 + - User label, for example ``SOMA``. + * - ``Path{n}/roi_###/mask`` + - rows x cols + - uint8 + - Binary ROI mask in image coordinates (1 = included pixel). + * - ``Path{n}/roi_###/position`` + - nVertices x 2 + - double + - Polygon vertices ``[y_loc, x_loc]`` when ``coords_zero_indexed=1``; + otherwise legacy ``[x, y]``. + * - ``Path{n}/roi_###/center`` + - 1 x 2 + - double + - Circle/ellipse center ``[y_loc, x_loc]`` when + ``coords_zero_indexed=1``; otherwise legacy ``[x, y]``. + * - ``Path{n}/roi_###/semi_axes`` + - 1 x 2 + - double + - Ellipse semi-axis lengths (ellipse only). + * - ``Path{n}/roi_###/rotation_angle`` + - 1 x 1 + - double + - Ellipse rotation angle in degrees (ellipse only). + * - ``Path{n}/roi_###/radius`` + - 1 x 1 + - double + - Circle radius (circle only). + +Experiment summary fields +~~~~~~~~~~~~~~~~~~~~~~~~~ + +Pixel/plane coordinates are **0-indexed** ``[z_loc, y_loc, x_loc]``; +``frame_line_idxs`` remains **1-indexed**. The Python writer emits every +dataset in the summary tree, including ``SNR`` and ``offlineZshifts``. +External summaries may omit SNR or Z shifts when unavailable. Missing model +SNR values are serialized as NaN. + +.. list-table:: ``experiment_summary.h5`` + :header-rows: 1 + :widths: 30 22 10 38 + + * - Field + - Size + - Data type + - Description + * - ``row_major`` + - 1 x 1 + - uint8 + - Layout flag. + * - ``params`` + - — + - group/struct + - Analysis parameters. External MATLAB ``activityChannel``, if present, + is 1-indexed into ``numChannels``. Python activity extraction uses + channel zero, not this reference selector. + * - ``Path{n}`` + - — + - group + - One group per imaging path. + * - ``Path{n}/Z_depths`` + - fastz x 1 + - numeric + - Z depths per imaging plane (SLAP2). + * - ``Path{n}/frame_info/offlineXshifts`` + - total frames x 1 + - numeric + - Offline registration X displacement per frame, using the public + alignment sign convention above. + * - ``Path{n}/frame_info/offlineYshifts`` + - total frames x 1 + - numeric + - Offline registration Y displacement per frame. + * - ``Path{n}/frame_info/offlineZshifts`` + - total frames x 1 + - numeric + - Offline registration Z displacement per frame. Emitted by Python + BandSILo; optional in the external schema when 3D alignment is absent. + * - ``Path{n}/frame_info/onlineXshifts`` + - total frames x 1 + - numeric + - SLAP2 online X shift per frame. + * - ``Path{n}/frame_info/onlineYshifts`` + - total frames x 1 + - numeric + - SLAP2 online Y shift per frame. + * - ``Path{n}/frame_info/onlineZshifts`` + - total frames x 1 + - numeric + - SLAP2 online Z shift per frame. + * - ``Path{n}/frame_info/trial_num_frames`` + - trials x 1 + - integer + - Number of frames contributed by each analysis trial. + * - ``Path{n}/frame_info/frame_line_idxs`` + - total frames x 1 + - integer + - Raw line index (SLAP2) or frame index (other microscopes in the + external schema) for each stitched-series frame; 1-indexed. + * - ``Path{n}/frame_info/discard_frames`` + - total frames x 1 + - bool or uint8 + - Frame excluded from analysis, for example by motion censoring. + * - ``Path{n}/visualizations/mean_im`` + - channels x fastz x rows x cols + - numeric + - Mean registered image per channel / Z slice. + * - ``Path{n}/visualizations/act_im`` + - fastz x rows x cols + - numeric + - Activity / localization summary image. + * - ``Path{n}/visualizations/act_im_peaks`` + - sources x 3 + - numeric + - Activity-image peak locations ``[z_loc, y_loc, x_loc]``, 0-indexed; + Python uses source row/column coordinates with ``z_loc`` fixed at ``0``. + * - ``Path{n}/global/F`` + - channels x total frames + - numeric + - Whole-field fluorescence traces, one row per channel. + * - ``Path{n}/user_rois/labels`` + - rois x 1 + - string + - User-defined ROI labels (UTF-8 strings in Python summary output). + * - ``Path{n}/user_rois/mask`` + - rois x fastz x rows x cols + - uint8 or bool + - Stacked binary masks for each user ROI. + * - ``Path{n}/user_rois/Fsvd`` + - rois x channels x total frames + - numeric + - ROI SVD / projection signals. Emitted as NaN unless explicitly + supplied through the model; not computed by BandSILo or copied from ``F``. + * - ``Path{n}/user_rois/F`` + - rois x channels x total frames + - numeric + - ROI fluorescence (raw or baseline-corrected according to the producer). + * - ``Path{n}/user_rois`` attribute ``annotation_enabled`` + - scalar attribute + - bool + - Python annotation policy: distinguishes disabled from enabled-but-empty + annotations. Both produce zero-ROI ``labels``, ``mask``, ``Fsvd``, and + ``F`` datasets with the remaining dimensions intact. + * - ``Path{n}/sources/spatial/profiles`` + - sources x fastz x rows x cols + - numeric + - Spatial component / pixel weights per source, averaged across trials + with footprints. + * - ``Path{n}/sources/spatial/coords`` + - sources x 3 + - numeric + - Source centers, one ``[z_loc, y_loc, x_loc]`` per row, 0-indexed. + * - ``Path{n}/sources/temporal/dF_ls`` + - sources x channels x total frames + - numeric + - Least-squares fluorescence change (absolute or scaled); Python uses + baseline re-estimation and processes activity channel zero only. + * - ``Path{n}/sources/temporal/dF_denoised`` + - sources x channels x total frames + - numeric + - Fitted convolution fluorescence change, saved alongside least squares + without a second baseline subtraction. Missing optional model data + serialize as NaN. + * - ``Path{n}/sources/temporal/events`` + - sources x channels x total frames + - numeric + - Nonnegative fitted event amplitudes; excludes each trial's initial + state. Discarded frames and non-activity channels are NaN. + * - ``Path{n}/sources/temporal/F0`` + - sources x channels x total frames + - numeric + - Baseline estimate used for normalization; based on QR, not the + transient model. + * - ``Path{n}/sources/temporal/SNR`` + - sources x 1 + - numeric + - Per-source signal-to-noise ratio metric. Python emits this dataset + (NaN for missing model values); optional in external summaries. + +Per-trial summary fields: unsupported external reference +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +**Python does not emit this file.** The reference trial axis includes all +analysis trials in the trial table. Trials without alignment or extraction +data have NaN slices. Coordinates use **0-indexed** +``[z_loc, y_loc, x_loc]``. + +.. list-table:: ``per_trial_summary.h5`` (external-reference only) + :header-rows: 1 + :widths: 30 22 10 38 + + * - Field + - Size + - Data type + - Description + * - ``row_major`` + - 1 x 1 + - uint8 + - Layout flag. + * - ``Path{n}`` + - — + - group + - One group per imaging path. + * - ``Path{n}/visualizations/per_trial_mean_im`` + - trials x channels x fastz x rows x cols + - numeric + - Trial-aligned mean registered image per channel / Z slice. + * - ``Path{n}/visualizations/per_trial_act_im`` + - trials x fastz x rows x cols + - numeric + - Trial-aligned activity / localization summary image. + * - ``Path{n}/visualizations/per_trial_act_im_peaks`` + - trials x max_peaks x 3 + - numeric + - Per-trial detected peak locations ``[z_loc, y_loc, x_loc]``, 0-indexed; + NaN-padded when a trial has fewer than ``max_peaks``. + * - ``Path{n}/visualizations/per_trial_num_peaks`` + - trials x 1 + - integer + - Number of valid peaks per trial. + * - ``Path{n}/sources/spatial/per_trial_profiles`` + - trials x sources x fastz x rows x cols + - numeric + - Spatial component / pixel weights per source per trial. + * - ``Path{n}/sources/spatial/per_trial_coords`` + - trials x sources x 3 + - numeric + - Source centers per trial, ``[z_loc, y_loc, x_loc]``, 0-indexed. + * - ``Path{n}/sources/temporal/per_trial_SNR`` + - trials x sources + - numeric + - Per-source SNR for each analysis trial. \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst index 4830f88..f9a210b 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,26 +1,33 @@ GIAnT-Python ============ -The implemented workflow is SLAP2 BandSILo extraction and ROI annotation -from a **pre-registered trial table**. Organization, registration, standard -SILo and deconvolution are not implemented; their placeholders have been removed. -Existing raw recordings, reference TIFFs, alignment files and the band -registration lookup table are required; this is not a full GIAnT-MATLAB -pipeline replacement. +GIAnT-Python is a Python package for processing glutamate imaging data, +with advanced capabilities for SLAP2 recordings. Currently, only **SLAP2 +Band Scan processing** is implemented: **BandSILo source extraction and +Band Scan ROI annotation** from a pre-registered trial table. + +Additional functionality is available in +`GIAnT-MATLAB `_, +and efforts are underway to port it into GIAnT-Python. Organization, +registration, standard SILo and general-purpose deconvolution stages are +not yet implemented in Python. The current workflow requires existing raw +recordings, reference TIFFs, alignment files and the band registration +lookup table. .. toctree:: :maxdepth: 2 :caption: Contents: - architecture migration + traces + file_formats + architecture -Standalone extraction ---------------------- +Getting started +--------------- Both an exact trial-table filename and a loaded ``TrialTable`` are accepted. -The filename is never guessed from the results directory. For example, given -``table_or_filename`` supplied by the caller: +Given ``table_or_filename`` supplied by the caller: .. code-block:: python @@ -36,106 +43,33 @@ The filename is never guessed from the results directory. For example, given annotations=AnnotationOptions(enabled=False, interactive=False), ) -For a chainable session using the same extraction service: - -.. code-block:: python - - from giant_python import ( - AnnotationOptions, BandSiloParams, ExecutionOptions, Pipeline, - ) - - pipeline = Pipeline.from_trial_table( - table_or_filename, - params=BandSiloParams(), - execution=ExecutionOptions(max_workers=6), - annotations=AnnotationOptions(enabled=False, interactive=False), - ).extract() - summary = pipeline.summary - -The band signature is -``extract_band_sources(input, params=None, *, execution=None, annotations=None)``. -``Pipeline`` also uses ``params=...`` for science. Each option scope accepts -only its corresponding dataclass, a dictionary of its own fields, or ``None``. -``resolve_band_options`` returns a fresh validated triple; mixed dictionaries -and cross-scope fields are rejected, not translated into other option scopes. -Explicit ``microscope`` / ``scan_mode`` routing belongs to the generic services -and ``Pipeline``; only SLAP2 band is implemented (``scan_mode=None`` also selects -band). Unsupported standalone routing raises ``ValueError``, not -``NotImplementedError``. - -``Pipeline`` exposes ``from_trial_table``, ``annotate`` and ``extract``; -``from_config``, ``organize``, ``register`` and ``run_all`` have been removed. -There are no constructor options/attributes named ``save_dir`` or -``align_params``. Directory overrides belong on the loaded trial table. -The ``giant`` CLI has only ``annotate`` and ``extract`` subcommands, with -``--microscope slap2`` and ``--scan-mode band`` as its sole routing choices. -Unsupported commands or choices fail argument parsing. - -An explicit ``annotate_rois(...)`` call or ``Pipeline.annotate()`` loads -existing annotations or draws them when permitted. For headless extraction -with existing ROIs, use ``AnnotationOptions(enabled=True, interactive=False)``; -missing required annotations fail rather than silently disabling ROIs. +For a chainable session, use ``Pipeline.from_trial_table(...).extract()`` and +read ``pipeline.summary``. The ``giant`` CLI offers ``annotate`` and ``extract``. +For headless extraction with existing ROIs, use +``AnnotationOptions(enabled=True, interactive=False)``; missing required +annotations fail explicitly. Output is written beneath the loaded table's ``savedr`` in ``source_extraction/experiment_summary.h5``. Copying a table does not rewrite -its stored directories. The repository example ``examples/extract_band.py`` -stages inputs and overrides ``datadr``/``savedr`` on a loaded model explicitly. -It refuses existing output directories and never changes the environment at -import time. See :doc:`migration` for canonical owners, data conventions and -deployment notes. The preceding Python API was unreleased and unused; -compatibility-only imports and adapters are removed, not supported aliases. - -Results and parameter editing ------------------------------ - -Use ``summary.paths`` exclusively for per-path data. Each ``PathSummary`` has -its own sources, ROIs, visualizations, frame information, depths and global -fluorescence. ``path_index`` is zero-based; disk groups begin at ``Path1``. -Access ``FrameInfo`` fields as attributes or use its shallow ``to_dict()``; -there is no mapping adapter or flat summary convenience API. - -``giant_python.gui.parameters_gui.run_parameter_gui(params=None, *, execution=None, -annotations=None)`` returns a validated ``(BandSiloParams, ExecutionOptions, -AnnotationOptions)`` triple, or ``None`` on cancellation, without mutating inputs. -Toolkit imports remain lazy. - -Supported schema subset ------------------------ - -The repository README preserves extensive MATLAB reference trees and field -tables. They are reference material, **not** a claim that Python emits every -listed field. Band output includes source profiles/coordinates, ``dF_ls``, -``F0``, ``SNR``, visualizations, depths, global fluorescence, frame information -and optional user ROI masks/labels/``F``. - -``dF_denoised``, ``events``, ROI ``Fsvd`` and the entire -``per_trial_summary.h5`` are future/reference-only and **not emitted**. -``ExperimentSummary.from_h5`` and ``to_h5`` support the emitted band subset -eagerly. Generic HDF5 decoding and parameter-group writing are shared working -helpers. ``TrialTable.to_h5`` and the ``AlignmentData`` model have been removed; -the raw band alignment reader remains implemented. Native TIFF/SLAP2 readers, -annotation persistence and summary writing are also retained. - -The summary codec is model-only: ``giant_python.io.read_summary(path)`` returns -an ``ExperimentSummary`` and ``giant_python.io.write_summary(summary, path)`` -returns ``None``. Band assembly uses -``giant_python.extraction.band.result_assembly.assemble_path_summary`` to construct -``PathSummary`` directly; no public assembled-dictionary writer remains. -Removing Python compatibility surfaces does not change MATLAB layout handling, -the exact ``Z_depths`` field spelling, coordinate conversions or shift signs. - -Placeholder removal — 2026-09-20 --------------------------------- - -The earlier requirement to retain future placeholders is superseded. All -unimplemented placeholder modules/functions and corresponding exports have -been removed, including ``AlignParams``, future pipeline stages, generic -numerical stubs and unsupported IO wrappers. Working band science, peak fitting, -the band-specific NMF solver and HDF5 codecs are unchanged. ``Slap2Reader`` is -retained as a typing protocol for real vendor readers, not a placeholder stage. -The README's MATLAB schemas remain external-format reference documentation. -See :doc:`migration` for the removal inventory and :doc:`architecture` for -current owners. +its stored directories. Use a **fresh output location**: summary updates are +not atomic replacement or resume. See :doc:`migration` for configuration, +the results API, parameter editing and safe staging with the repository's +``examples/extract_band.py``. + +Current outputs +--------------- + +``summary.paths`` holds eager per-path sources, ROIs, visualizations, frame +information, depths and global fluorescence. BandSILo emits ``dF_ls``, +``dF_denoised`` and ``events``, together with source profiles, coordinates, +``F0`` and ``SNR``. ROI ``Fsvd`` is written as NaN unless supplied through the +model. Disabled or enabled-but-empty annotations produce zero-ROI datasets; +the ``user_rois`` group's ``annotation_enabled`` attribute distinguishes them. + +See :doc:`traces` for estimation and interpretation, :doc:`file_formats` for +schemas and indexing, and :doc:`architecture` for module ownership and +numerical boundaries. External reference formats, including per-trial +summaries and registration products, do not imply Python output support. Public API reference -------------------- @@ -173,6 +107,8 @@ modules. .. autoclass:: giant_python.models.trial_table.TrialTable :members: from_h5 +.. autoclass:: giant_python.models.trial_table.Slap2Info + .. automodule:: giant_python.io.slap2 :members: Slap2Reader, open_slap2_file, load_alignment_data_h5 @@ -182,8 +118,7 @@ modules. .. automodule:: giant_python.io.hdf5 :members: load_struct_h5, write_dict_to_h5group -The package export lists below include only the current public surfaces; -removed placeholder names are not aliases or deprecated entry points. +Package exports: .. autodata:: giant_python.__all__ diff --git a/docs/source/migration.rst b/docs/source/migration.rst index 6b576a5..1f257f8 100644 --- a/docs/source/migration.rst +++ b/docs/source/migration.rst @@ -4,36 +4,31 @@ API and data conventions Public workflow and options --------------------------- -Use the lazy package-root exports ``extract_band_sources``, -``extract_sources``, ``annotate_rois``, ``Pipeline``, ``TrialTable``, -``ExperimentSummary``, ``BandSiloParams``, ``ExecutionOptions`` and -``AnnotationOptions``. Both extraction functions accept an exact trial-table -filename or a loaded model. Do not reconstruct a filename from ``savedr``. -``Pipeline.from_trial_table(..., params=BandSiloParams()).extract()`` returns -the same pipeline; read the extraction result through ``pipeline.summary``. +The lazy package-root exports provide ``extract_band_sources``, +``extract_sources``, ``annotate_rois``, ``Pipeline`` and the input, option and +result models. Both extraction functions accept an exact trial-table filename +or a loaded ``TrialTable`` and return ``ExperimentSummary``. +``Pipeline.from_trial_table(...).extract()`` returns the same pipeline; +read its result through ``pipeline.summary``. ``Pipeline.annotate()`` and +``annotate_rois(...)`` load existing annotations or draw them when permitted. The band service signature is ``extract_band_sources(input, params=None, *, execution=None, annotations=None)``. -``Pipeline(..., params=...)`` uses the same scientific option scope. The chain -method is ``extract()``; ``extract_sources()`` is a standalone generic service, -not a second chain method. Generic extraction and annotation services, and -``Pipeline``, take explicit ``microscope`` / ``scan_mode`` routing arguments; -only ``microscope='slap2'`` with band mode is implemented. ``scan_mode=None`` -also selects band. Unsupported routing through standalone services raises -``ValueError`` before input/backend loading, not ``NotImplementedError``. - -``Pipeline`` has only ``from_trial_table``, ``annotate`` and ``extract`` as its -public workflow methods. ``from_config``, ``organize``, ``register`` and -``run_all`` are removed, together with constructor options/attributes -``save_dir`` and ``align_params``. Use the loaded table's ``datadr`` / ``savedr`` -for explicit input/output directory overrides. Missing loaded state raises -``ValueError`` when an operation is requested. - -The ``giant`` CLI offers only ``annotate`` and ``extract``, each with a trial-table -filename. Its only microscope choice is ``slap2`` and its only scan-mode choice -is ``band``. Unsupported commands/options are argument errors (exit code 2). -Use ``--draw-user-rois --headless`` to extract with existing annotations without -drawing; missing required annotations fail explicitly. +``Pipeline`` uses the same option scopes. Generic services and ``Pipeline`` +take explicit ``microscope`` / ``scan_mode`` routing arguments; only +``microscope='slap2'`` with ``scan_mode='band'`` (or ``None``) is supported. +Unsupported standalone routing raises ``ValueError`` before input/backend +loading. Pipeline operations require a loaded table. + +The ``giant`` CLI offers ``annotate`` and ``extract``, each with a trial-table +filename and ``slap2`` / ``band`` routing choices. Invalid commands/options +are argument errors (exit code 2). For existing headless ROIs use +``--draw-user-rois --headless``, or in Python +``AnnotationOptions(enabled=True, interactive=False)``. Missing required +annotations fail explicitly rather than silently disabling ROIs. + +Configuration +------------- .. list-table:: Independent configuration scopes :header-rows: 1 @@ -55,123 +50,47 @@ drawing; missing required annotations fail explicitly. fresh validated objects in the order shown above. Each input accepts **only its corresponding dataclass, a dictionary of that scope's fields, or None**. Mixed dictionaries, cross-scope fields, unknown keys and unrelated dataclasses -are rejected; fields are not moved between scopes or converted from a combined -carrier. Caller-owned inputs are not mutated, even when acquisition channel -counts or GUI policy are later resolved. +raise ``TypeError``. Caller-owned inputs are not mutated, even when acquisition +channel counts or GUI policy are later resolved. -Defaults remain peak threshold 7, analysis rate 100 Hz, decay 0.15 seconds, +Defaults include peak threshold 7, analysis rate 100 Hz, decay 0.15 seconds, baseline window 4 seconds, denoise window 1 second, VIF 1.38 and PSF dilation 17. Execution defaults to six workers, no trial limit and verbosity off. Annotation -defaults to disabled with automatic interactivity. -``operator`` has been removed from ``AnnotationOptions``, the parameter GUI, -the CLI and the staging example. Supplying it as an annotation option raises -``TypeError``; ``--operator`` is no longer accepted. New extraction summaries -do not emit operator metadata; existing summary metadata remains readable. -``num_channels`` and ``activity_channel`` have been removed from -``BandSiloParams``. Channel count is inferred from acquisition metadata and -retained in output metadata; source activity extraction always uses channel -zero (zero-based). Supplying either removed option raises ``TypeError``. -If channel count cannot be inferred, extraction fails with ``ValueError``; -provide acquisition metadata for a kept trial rather than a manual override. -``AlignParams`` has been removed; registration has no Python configuration API. - -Canonical owner map -------------------- - -The preceding Python API was unreleased and unused. Compatibility-only -namespaces, combined options, extraction adapters and result/IO aliases have -been removed, not deprecated. No old Python import support or production -Python migration procedure is promised. MATLAB file-format compatibility is -retained independently of those removed Python surfaces. - -.. list-table:: Canonical modules under giant_python - :header-rows: 1 - :widths: 44 56 - - * - Owner - - Interface or responsibility - * - ``pipeline.extract`` / ``pipeline.annotate`` / ``pipeline.pipeline`` - - Root-exported extraction/annotation functions and ``Pipeline``; - common services also used by the CLI - * - ``models.params`` - - Split configuration and ``resolve_band_options``; no registration options - * - ``models.experiment_summary`` - - ``ExperimentSummary``, ``PathSummary``, ``FrameInfo``, ``Source``, - ``UserRoi`` and ``Visualizations`` - * - ``io.experiment_summary`` - - ``read_summary(path)`` / ``write_summary(summary, path)``; - also exported by ``io``; model-only summary interface - * - ``io.hdf5`` / ``io.trial_table`` - - ``load_struct_h5`` and parameter-group writing; ``read_trial_table`` - builds portable metadata and serves ``TrialTable.from_h5`` - * - ``io.slap2`` / ``io.tiff`` - - Raw acquisition/alignment reading and TIFF decoding, without private - band motion-sign normalization - * - ``extraction.band.inputs`` - - Runtime paths, kept trials, private reference offsets, - lookup/reference loading and PSF discovery (including ``default_psf``) - * - ``extraction.band.workflow`` / ``extraction.band.result_assembly`` - - Scientific orchestration; ``assemble_path_summary`` constructs the - public path model, with persistence delegated to shared IO - * - ``extraction.band.geometry`` / ``extraction.band.operators`` - - Pure pixel support, PSF padding/cropping and sparse projection operators - * - ``extraction.band.motion_binning`` / ``extraction.band.background`` / - ``extraction.band.activity`` - - Band motion policies, background estimation and activity calculations - * - ``extraction.band.localization`` - - Implemented band-specific NMF/localization; the generic NMF stub is removed - * - ``extraction.band.traces`` / ``extraction.band.summary_images`` / - ``extraction.band.trial_data`` - - Band trace extraction, image assembly and superpixel data reduction - * - ``numerics.baseline`` / ``numerics.peaks`` / ``numerics.interpolation`` / - ``numerics.morphology`` - - Shared implemented numerical kernels and helpers - * - ``gui.roi_editor`` / ``gui.parameters_gui`` - - Interactive presentation with lazy toolkit imports - * - ``io.annotations`` / ``extraction.band.rois`` / ``extraction.band.annotation`` - - Annotation persistence, pure band ROI geometry and annotation policy - * - ``parallel`` / ``progress`` - - Ordered trial mapping and opt-in logging/progress +defaults to disabled with automatic interactivity. See :doc:`traces` for +temporal-fit controls, motion-bin tolerance and random-seed semantics. -Placeholder removal — 2026-09-20 --------------------------------- +Channel count is inferred from acquisition metadata and retained in output +metadata. Source activity extraction always uses **channel zero**; other +source-activity channels contain NaNs. Neither channel count nor activity +channel is a scientific option. If channel count cannot be inferred, +extraction raises ``ValueError``; supply acquisition metadata for a kept trial. -The earlier requirement to retain future placeholders is explicitly superseded. -Unimplemented functions/files and corresponding exports are removed, not -deprecated or replaced by forwarding modules. This is an API-scope cleanup, -not an implementation of the omitted GIAnT-MATLAB stages. - -Removed modules under ``giant_python``: - -* ``numerics.deconv``, ``numerics.nmf``, ``numerics.registration`` and - ``numerics.variance`` (generic numerical stubs). -* ``pipeline.base``, ``pipeline.organize`` and ``pipeline.register`` (including - ``Stage``, ``TrialTableBuilder`` and ``MotionCorrector``). -* ``gui.draw_rois`` (not the working ROI editor) and ``models.alignment`` - (including ``AlignmentData``). - -Removed members in retained modules include ``AlignParams``, -``TrialTable.to_h5``, ``numerics.interpolation.interp_frame``, -``io.hdf5.save_struct_h5``, ``io.slap2.get_online_motion``, -``io.tiff.scanimagetiff_wrapper``, ``io.tiff.scanimagetiff_data_wrapper``, -and ``pipeline.extract.silo``. -The unsupported Pipeline methods/options listed above and their related exports -are also gone. Config-file construction, checkpoint/resume, organization, -registration, standard SILo, deconvolution and trial-table writing remain -unavailable; calling a removed API is not a supported capability check. - -Retained functionality includes ``nearest_interp``, peak detection/fitting, -baseline/morphology kernels, the working band NMF/localization solver, native -TIFF/SLAP2 readers, raw alignment decoding, ``write_dict_to_h5group``, annotation -persistence and experiment-summary codecs. ``Slap2Reader`` is a structural -typing protocol for real vendor readers, not a workflow stub to delete. -Scientific algorithms, defaults, ordering and external file conventions are -unchanged by placeholder removal. The README retains full MATLAB schema -descriptions, including unsupported fields and external registration outputs, -as reference documentation rather than promised Python functionality. - -Result, IO and GUI contracts ------------------------------- +``gui.parameters_gui.run_parameter_gui(params=None, *, execution=None, +annotations=None)`` returns a validated ``(BandSiloParams, ExecutionOptions, +AnnotationOptions)`` triple, or ``None`` on cancellation, without mutating +inputs. Its sparse-factor control is in natural-log units. GUI toolkit imports +are lazy; see :doc:`architecture` for module ownership and numerical boundaries. + +Input identity and output location +---------------------------------- + +``Pipeline.from_trial_table`` loads an exact filename once and retains it as +``trial_table_path``; a loaded table is retained directly. ``source_path`` is +provenance, not a filename reconstructed from ``savedr``. Set ``datadr`` and +``savedr`` on the loaded ``TrialTable`` for explicit directory overrides and +pass that model to extraction. Copying the table file does not rewrite its +stored directories. + +Extraction writes beneath ``savedr`` to +``source_extraction/experiment_summary.h5``. Use a fresh output location: +updating an existing summary replaces supplied path groups but retains +unrelated paths and root parameters/layout flags. Array preparation before +each path replacement does not make writes file-wide atomic or provide +checkpoint/resume. Unknown external fields need not survive a model round-trip. +See :doc:`file_formats` for the persistence contract. + +Results API +----------- All imaging results belong to ``ExperimentSummary.paths``. Each ``PathSummary`` owns its sources, ROIs, visualizations, ``FrameInfo``, depths and global @@ -182,78 +101,27 @@ is not necessarily that identity when reading selected paths. ``FrameInfo`` exposes attributes and a shallow ``to_dict()`` only, not mapping methods. Results are eager; per-source/ROI entries may share array storage. They are neither lazy HDF5 proxies nor independent immutable snapshots. -Disabled annotations omit the ROI group; enabled-but-empty is a distinct state. +Mutating shared storage can affect other views; serialization may stack +arrays and does not promise zero-copy writing. ``io.experiment_summary.read_summary(path)`` returns ``ExperimentSummary`` and ``write_summary(summary, path)`` accepts that model and returns ``None``. +Both functions are also exported by ``giant_python.io``. ``ExperimentSummary.from_h5`` / ``to_h5`` delegate to these canonical functions. -``extraction.band.result_assembly.assemble_path_summary`` directly builds -``PathSummary``; no public assembled-dictionary result adapter or writer remains. -The codec supports the emitted band subset, not the full reference schema. -It does not emit denoised traces, events, ROI SVD fluorescence or per-trial -summaries. New files set ``row_major=1``. MATLAB layout decoding remains -supported (missing or zero flag reverses multidimensional axes as needed). -Rank-two frame vectors, one-based acquisition line indexes, zero-based spatial -coordinates and annotation coordinate flags are unchanged. The HDF5 field -spelling is exactly ``Z_depths``; ``z_depths`` is the path model attribute. +Production summaries emit ``dF_ls``, ``dF_denoised`` and ``events`` alongside +source profiles, ``F0``, ``SNR``, visualizations, depths, global fluorescence +and frame information. ROI ``Fsvd`` is emitted as NaN unless explicitly +supplied through the model; BandSILo does not calculate it. Disabled and +enabled-but-empty annotations both emit zero-ROI datasets, distinguished by +the ``annotation_enabled`` attribute on the ``user_rois`` group. -Writing an existing summary replaces supplied path groups while retaining -unrelated paths and existing root parameters/layout flags. Required source/ROI -arrays are prepared before replacing each path, but the operation is not a -file-wide transaction or resume mechanism. Prefer a fresh destination; -unknown reference fields are not guaranteed a lossless model round-trip. - -``gui.parameters_gui.run_parameter_gui(params=None, *, execution=None, -annotations=None)`` returns ``(BandSiloParams, ExecutionOptions, -AnnotationOptions)`` or ``None`` on cancellation. The form retains its -natural-log sparse-factor control and returns validated options without -mutating inputs; it does not combine them into a legacy carrier. - -Gaussian parameter convention ------------------------------ - -``numerics.peaks.gaussian_peaks_integrated`` accepts only ``(N, 5)`` parameters -``[amp, mu_y, mu_x, sigma_y, sigma_x]``. Supply both widths explicitly, even -for isotropic Gaussians; there is no four-column adapter. -Peak fitting includes an isotropic warm start followed by anisotropic -refinement. Both stages retain five-column parameters; the first stage ties -width updates internally using their combined derivative, then the second -allows independent widths. Removing the alternate representation does not -retune scientific defaults or this implemented fitting policy. - -Motion sign and data provenance -------------------------------- - -Public ``FrameInfo.offlineXshifts``, ``offlineYshifts`` and -``offlineZshifts`` contain **alignment displacement**, matching standard -MATLAB SILo's saved sign. At matching sample locations this is the sign of -the original alignment ``motionDSc``, ``motionDSr`` and ``motionDSz``. -Displacement is added to reference/view-grid coordinates when sampling raw -images; mapping raw measurement positions into reference coordinates uses -its negative. These two mapping directions must not be conflated. - -The raw alignment reader preserves saved values. Band input normalization -negates offline displacement into private reference offsets. Band result -assembly in ``assemble_path_summary`` converts those offsets back exactly once -before building public ``PathSummary``/``FrameInfo`` objects or saving them. -The summary codec itself does not flip signs. No inter-trial XY correction is -invented where the band workflow has not computed one. - -The preceding Python version was never released or used, so no production -Python summary migration is required. **If an experimental file from an -unreleased checkout exists**, its offline sign needs producer/alignment-data -provenance. Do not auto-correct on read, use the installed package version -alone to infer file history, or infer provenance from zero-valued shifts. -Retain originals before any explicit transformation. MATLAB-produced and -already correct summaries must not be negated again. Online shifts are -unchanged by the Python API cleanup. - -This is sign compatibility, not a claim of identical interpolation: the band -code keeps its existing linear/clamped interpolation, whereas MATLAB's -standard summary builder uses PCHIP/extrapolation. Scientific defaults, -center/surround kernels, motion-bin ordering, numerical/random call order, -trial ordering, and internal extraction calculations are unchanged. +New summaries set ``row_major=1`` and preserve rank-two frame vectors. +Acquisition line indexes are one-based; summary spatial coordinates are +zero-based ``[z, y, x]``. ``PathSummary.z_depths`` maps to ``Z_depths`` on disk. +See :doc:`file_formats` for complete shapes, MATLAB layout decoding, annotation +coordinate flags and motion-sign provenance; see :doc:`traces` for temporal +output interpretation. External per-trial summaries are not Python outputs. Staging example --------------- @@ -269,13 +137,13 @@ selects directory linking (Windows may require Developer Mode or link privileges). ``--stage-data copy`` or ``symlink`` stages the complete raw directory, including sidecars and reference TIFFs; ``none`` uses it in place. This is staging of externally generated motion correction, not execution of -Python registration (which is not implemented and has no placeholder API). +Python registration. Crucially, the example overrides ``datadr`` and ``savedr`` on the loaded ``TrialTable`` and passes that model directly to extraction. It does not reload the copied file and lose those overrides. The copied file is an unaltered provenance artifact: its stored directories are not rewritten, -and there is no ``TrialTable.to_h5`` method to call. Input filename fields +and Python does not write trial tables. Input filename fields must be relative without parent traversal; absolute names are rejected rather than silently bypassing staging or being guessed from basenames. @@ -292,19 +160,9 @@ options are validated before any staging or environment changes. Its environment override is scoped to staging/extraction, before importing the lazy extraction service, and restored even when extraction fails. Without the flag, the existing environment and joblib defaults are left alone. -No fixed container paths or operator names are embedded. Dependencies must -already be installed; the example does not install packages. +Dependencies must already be installed; the example does not install packages. Scientific and execution flags are explicit run choices, not changes to library defaults. Use ``--use-rois --annotations-dir`` for existing headless annotations, or add ``--interactive`` to permit drawing. Inspect ``--help`` -for all flags. - -Dependency note ---------------- - -Verbose progress imports ``tqdm`` lazily with no missing-package fallback; -it is declared directly. The existing band editor uses OpenCV and tkinter, -not napari. Removing Python compatibility code is not a GUI redesign and -changes no numerical dependency, quality threshold, coverage exclusion or -execution backend. \ No newline at end of file +for all flags. \ No newline at end of file diff --git a/docs/source/traces.rst b/docs/source/traces.rst new file mode 100644 index 0000000..c7bb491 --- /dev/null +++ b/docs/source/traces.rst @@ -0,0 +1,118 @@ +Trace estimation +================ + +Production BandSILo extraction computes and saves least-squares +``sources/temporal/dF_ls``, convolution-constrained +``sources/temporal/dF_denoised``, and fitted ``sources/temporal/events``. +Each array has shape ``(sources, channels, total frames)``. Source activity +extraction uses **channel zero**; other channels and missing observations are +NaN. Both trace estimates are mandatory production outputs, not selectable +trace modes. Low-level loaded-array functions also support weighted averages +for diagnostics. See :doc:`file_formats` for the saved layout. + +Trial-wise objective +-------------------- + +The trace-only solver holds spatial profiles fixed and minimizes + +.. math:: + + \min_{s \geq 0}\; \frac{1}{2} + \left\|(d-b)-H_{\mathrm{motion}} A K s\right\|_2^2 + + \beta \sum_j \sum_{t=1}^{T-1} s_{j,t}. + +Here ``d`` is the observed data, ``b`` is the background, ``A`` contains the +fixed source profiles, ``H_motion`` is the motion-dependent observation +operator, and ``K`` applies a causal unit-peak exponential. For each source, +:math:`s_{j,0}` is an unpenalized nonnegative initial state; the remaining entries +are nonnegative event amplitudes. The solver operates chronologically over +each complete trial, including motion changes and missing frames. No state or +event inference crosses trial boundaries. Localization, NMF, and profile +fitting are separate from this trace-only optimization. + +Returned fluorescence changes preserve the QR complete-frame validity mask. +The standard least-squares trace and its background retain baseline +re-estimation for ``dF_ls`` and ``F0``; ``F0`` comes from QR, not the transient +model. ``dF_denoised`` is the fitted convolution trace itself, with no second +baseline subtraction. ``events`` excludes the initial state at each trial's +first frame and stores NaN for discarded frames. + +Controls and convergence +------------------------ + +``BandSiloParams.decay_tau_s`` controls both the trace kernel and localization +filtering. Its default is 0.15 seconds, and it is saved as ``decayTau_s``. +The GUI's Decay Tau field and CLI ``--decay-tau-s`` flag change this shared +setting; there is no separate trace decay parameter. + +.. list-table:: Trace-only controls + :header-rows: 1 + :widths: 28 12 60 + + * - Parameter + - Default + - Meaning + * - ``trace_beta_fraction`` + - ``0.01`` + - Fraction of each trial's positive adjoint scale; zero disables L1. + * - ``trace_max_iter`` + - ``500`` + - Maximum L-BFGS-B iterations per optimization stage. + * - ``trace_tolerance`` + - ``1e-7`` + - Objective/gradient tolerance. + * - ``trace_debias`` + - ``False`` + - Refit selected events without L1 shrinkage. + * - ``trace_event_threshold`` + - ``1e-6`` + - Absolute event-amplitude threshold for debias support. + +These controls are recorded in summary parameters and are available as +hyphenated CLI flags (for example, ``--trace-beta-fraction``). +A nonconverged production fit raises an error rather than saving an unfinished +estimate. Increase the iteration limit or inspect a low-level fit when this +occurs. Per-trial optimizer diagnostics and latent in-gap predictions are not +saved in the summary. + +Motion selection and reproducibility +------------------------------------ + +``BandSiloParams(z_tol=1.5)`` sets the low-resolution motion-bin selection +tolerance in microns. Bins must lie within this absolute distance of the +median frame z and satisfy the minimum frame count. The value must be positive +and finite. The CLI and staging example expose ``--z-tol``; the parameter GUI +labels it **Z Tolerance (µm)**. + +``BandSiloParams(random_seed=0)`` seeds the random motion-bin order used by +source localization (NMF and Adam). It must be a nonnegative integer. Each +path and outer iteration uses its own generator derived from this seed, +independently of the global torch RNG. The seed is saved in summary parameters +and exposed as ``--random-seed``. It fixes random draws, not floating-point +results across hardware, thread counts, or package versions. + +Loaded-array API +---------------- + +For loaded arrays, pass ``transient_options=TransientOptions(...)`` and +``analyze_hz=...`` to +``giant_python.extraction.band.traces.compute_high_res_traces``. +``TransientOptions`` and ``fit_transient_traces`` are defined in +``giant_python.extraction.band.transient``. The latter accepts an absolute +``beta`` or an explicit causal kernel and exposes events, the initial state, +in-gap model predictions, actual beta, and convergence diagnostics. +Retain the complete chronological output grid, including gaps, when supplying +arrays. Low-level nonconvergence warns and returns diagnostics; callers must +check ``metadata['converged']`` before saving a result. +``TrialTraceResult`` retains its original eight fields in order and appends +``d_f_denoised`` and ``events``. + +Scientific interpretation +------------------------- + +This is an output-grid estimator: overlapping acquisition-line averaging is +not modeled exactly. In-gap predictions are not observations, and glutamate +events are not automatically neuronal spikes. A smoother trace alone does not +establish improved SNR. Use a fresh output location when comparing estimates +or settings; summary updates are not an atomic replacement or a resume +mechanism. \ No newline at end of file diff --git a/src/giant_python/cli.py b/src/giant_python/cli.py index 132d496..c22007c 100644 --- a/src/giant_python/cli.py +++ b/src/giant_python/cli.py @@ -1,15 +1,8 @@ -"""Command-line interface: ``giant annotate|extract``. - -Thin wrapper over the pipeline stages for batch / cluster use. Wired as the -``giant`` console script in pyproject.toml. The ``annotate`` subcommand is the -standalone ROI-annotation step (the BandSILo analog of the GIAnT-MATLAB -annotation capsule): it writes ``annotations.h5``, which a later ``extract`` -run consumes without opening a GUI. - -CLI, standalone functions and Pipeline share the same workflow services. -Scientific defaults are unchanged; execution and annotation flags build -separate options. Heavy acquisition and GUI modules are imported only by -the selected backend when needed. +"""CLI for annotation and extraction from registered SLAP2 band inputs. + +``giant annotate`` loads or draws ROIs; ``giant extract`` runs BandSILo. +Both delegate to the public workflow services with separate scientific, +execution and annotation options. GUI imports occur only when needed. """ from __future__ import annotations diff --git a/src/giant_python/extraction/band/activity.py b/src/giant_python/extraction/band/activity.py index 959abfc..b302c0f 100644 --- a/src/giant_python/extraction/band/activity.py +++ b/src/giant_python/extraction/band/activity.py @@ -68,7 +68,7 @@ def _valid_sel_cols_for_motion( ) valid_pix_mask = ndimage.binary_erosion( valid_pix_mask, - # TODO: check if dimensions here should change + # Erode each plane with PSF-height rows and (2 * PSF-width - 1) columns. structure=np.ones( (1, psf2d.shape[0], psf2d.shape[1] * 2 - 1), dtype=bool ), diff --git a/src/giant_python/extraction/band/localization.py b/src/giant_python/extraction/band/localization.py index 157161f..6608717 100644 --- a/src/giant_python/extraction/band/localization.py +++ b/src/giant_python/extraction/band/localization.py @@ -690,7 +690,7 @@ def compute_source_snr( var_residual[j] += torch.sum( torch.sum(residual_recon[contributing][:, frames], dim=0) ** 2 ) - # TODO: maybe consider making this a peak SNR ratio + # Ratio of accumulated squared signals, not a peak-amplitude SNR. return var_exp / var_residual diff --git a/src/giant_python/extraction/band/operators.py b/src/giant_python/extraction/band/operators.py index 8161397..492affd 100644 --- a/src/giant_python/extraction/band/operators.py +++ b/src/giant_python/extraction/band/operators.py @@ -17,8 +17,8 @@ def expand_psf( """Normalize a PSF and build a center-aligned higher-resolution copy. The expanded PSF is an ``ex_fac``x up-sampled cubic-spline interpolation - that keeps the center aligned, used to form the ``HD - HD_expanded`` - matched filter in :func:`compute_rho`. + that keeps the center aligned. Localization constructs its center/surround + filter separately with :func:`gaussian_kernel_2d`. Parameters ---------- From 1549438136316c6990debb2bfaaeaa56312d51b8 Mon Sep 17 00:00:00 2001 From: m-xie Date: Thu, 22 Oct 2026 22:39:20 +0000 Subject: [PATCH 52/55] clean up some unused parameters in background estimation --- src/giant_python/extraction/band/workflow.py | 7 ------- tests/test_bandsilo_pipeline.py | 7 ++----- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/src/giant_python/extraction/band/workflow.py b/src/giant_python/extraction/band/workflow.py index 98a75e5..c8daae7 100644 --- a/src/giant_python/extraction/band/workflow.py +++ b/src/giant_python/extraction/band/workflow.py @@ -295,8 +295,6 @@ def _read(trial_ix): num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row, - psf2d, - num_super_pixels, align_hz, params, execution=execution, @@ -418,8 +416,6 @@ def _estimate_background( num_fast_zs, dmd_pixels_per_column, dmd_pixels_per_row, - psf2d, - num_super_pixels, align_hz, params, *, @@ -428,9 +424,6 @@ def _estimate_background( """Compose interpolation and rolling baseline on the original grid.""" execution = ExecutionOptions() if execution is None else execution log("Estimating background", execution.verbose) - # Preserve the original preparation order, even though these kernels are - # not consumed by the interpolation-based background reconstruction. - operators.expand_psf(psf2d) interp_data = np.full( (sel_pix_idxs.shape[0], low_res_data_norm.shape[1]), np.nan, diff --git a/tests/test_bandsilo_pipeline.py b/tests/test_bandsilo_pipeline.py index c78dba6..cfc3016 100644 --- a/tests/test_bandsilo_pipeline.py +++ b/tests/test_bandsilo_pipeline.py @@ -312,7 +312,7 @@ def test_persisted_trace_settings_exclude_removed_mode_keys(self): self.assertEqual(metadata[name], getattr(params, name)) def test_default_execution_skips_empty_planes_and_preserves_grid(self): - """Interpolate selected planes; baseline and assembly share data.""" + """Interpolate without PSF expansion; preserve the background grid.""" data = np.array([[3.0, 4.0], [5.0, 6.0]], dtype=np.float32) selected = np.array([4, 5]) ref_d, ref_r, ref_c = ( @@ -356,7 +356,6 @@ def test_default_execution_skips_empty_planes_and_preserves_grid(self): wf.bg, "assemble_background", return_value=assembled ) ) - psf = np.ones((3, 3)) result = wf._estimate_background( data, umyx, @@ -368,12 +367,10 @@ def test_default_execution_skips_empty_planes_and_preserves_grid(self): 2, 2, 2, - psf, - 2, 10, params, ) - expand.assert_called_once_with(psf) + expand.assert_not_called() self.assertEqual(planes.call_count, 2) interpolate.assert_called_once() np.testing.assert_array_equal(interpolate.call_args.args[0], data) From 7cffab57d837c8586d9f716c2b48ff3df039ad16 Mon Sep 17 00:00:00 2001 From: m-xie Date: Thu, 22 Oct 2026 23:04:49 +0000 Subject: [PATCH 53/55] remove unvetted unit tests --- tests/__init__.py | 1 - tests/test_acquisition_boundaries.py | 215 ------- tests/test_band_workflow.py | 587 ----------------- tests/test_bandsilo_annotate.py | 227 ------- tests/test_bandsilo_background.py | 612 ------------------ tests/test_bandsilo_baseline.py | 130 ---- tests/test_bandsilo_geometry.py | 179 ------ tests/test_bandsilo_gui.py | 232 ------- tests/test_bandsilo_hdf5.py | 308 --------- tests/test_bandsilo_nmf.py | 584 ----------------- tests/test_bandsilo_peaks.py | 726 --------------------- tests/test_bandsilo_phase_filter.py | 361 ----------- tests/test_bandsilo_pipeline.py | 851 ------------------------- tests/test_bandsilo_progress.py | 44 -- tests/test_bandsilo_summary_images.py | 441 ------------- tests/test_bandsilo_traces.py | 665 ------------------- tests/test_bandsilo_transient.py | 169 ----- tests/test_bandsilo_trial_data.py | 299 --------- tests/test_cli.py | 369 ----------- tests/test_configuration.py | 266 -------- tests/test_example.py | 17 - tests/test_experiment_summary_codec.py | 405 ------------ tests/test_io.py | 94 --- tests/test_io_architecture.py | 281 -------- tests/test_io_edges.py | 523 --------------- tests/test_models.py | 165 ----- tests/test_motion_convention.py | 89 --- tests/test_no_placeholders.py | 335 ---------- tests/test_numeric_architecture.py | 296 --------- tests/test_numeric_edges.py | 184 ------ tests/test_package_metadata.py | 149 ----- tests/test_parallel.py | 47 -- tests/test_parameter_gui.py | 166 ----- tests/test_path_summary.py | 420 ------------ tests/test_pipeline.py | 82 --- tests/test_staging_example.py | 559 ---------------- tests/test_workflows.py | 320 ---------- 37 files changed, 11398 deletions(-) delete mode 100644 tests/__init__.py delete mode 100644 tests/test_acquisition_boundaries.py delete mode 100644 tests/test_band_workflow.py delete mode 100644 tests/test_bandsilo_annotate.py delete mode 100644 tests/test_bandsilo_background.py delete mode 100644 tests/test_bandsilo_baseline.py delete mode 100644 tests/test_bandsilo_geometry.py delete mode 100644 tests/test_bandsilo_gui.py delete mode 100644 tests/test_bandsilo_hdf5.py delete mode 100644 tests/test_bandsilo_nmf.py delete mode 100644 tests/test_bandsilo_peaks.py delete mode 100644 tests/test_bandsilo_phase_filter.py delete mode 100644 tests/test_bandsilo_pipeline.py delete mode 100644 tests/test_bandsilo_progress.py delete mode 100644 tests/test_bandsilo_summary_images.py delete mode 100644 tests/test_bandsilo_traces.py delete mode 100644 tests/test_bandsilo_transient.py delete mode 100644 tests/test_bandsilo_trial_data.py delete mode 100644 tests/test_cli.py delete mode 100644 tests/test_configuration.py delete mode 100644 tests/test_example.py delete mode 100644 tests/test_experiment_summary_codec.py delete mode 100644 tests/test_io.py delete mode 100644 tests/test_io_architecture.py delete mode 100644 tests/test_io_edges.py delete mode 100644 tests/test_models.py delete mode 100644 tests/test_motion_convention.py delete mode 100644 tests/test_no_placeholders.py delete mode 100644 tests/test_numeric_architecture.py delete mode 100644 tests/test_numeric_edges.py delete mode 100644 tests/test_package_metadata.py delete mode 100644 tests/test_parallel.py delete mode 100644 tests/test_parameter_gui.py delete mode 100644 tests/test_path_summary.py delete mode 100644 tests/test_pipeline.py delete mode 100644 tests/test_staging_example.py delete mode 100644 tests/test_workflows.py diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index 816e430..0000000 --- a/tests/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""Testing library""" diff --git a/tests/test_acquisition_boundaries.py b/tests/test_acquisition_boundaries.py deleted file mode 100644 index f2921a8..0000000 --- a/tests/test_acquisition_boundaries.py +++ /dev/null @@ -1,215 +0,0 @@ -"""Guard the band acquisition boundary without changing numerical policy.""" - -import ast -import inspect -import tempfile -import unittest -from importlib import resources -from pathlib import Path -from unittest.mock import call, patch - -import h5py -import numpy as np -import tifffile - -from giant_python.extraction.band import annotation, geometry, inputs, workflow -from giant_python.io import slap2 -from giant_python.io.tiff import read_tiff - - -class TestAcquisitionOwnership(unittest.TestCase): - """File loading cannot leak back into geometry or duplicate raw IO APIs.""" - - def test_geometry_imports_only_numerical_support(self): - """Inspect all imports, including deferred imports inside functions.""" - tree = ast.parse(inspect.getsource(geometry)) - allowed = {"__future__", "typing", "numpy", "scipy"} - for node in ast.walk(tree): - if isinstance(node, ast.Import): - for alias in node.names: - self.assertIn(alias.name.split(".")[0], allowed) - elif isinstance(node, ast.ImportFrom): - self.assertEqual(node.level, 0) - self.assertIn((node.module or "").split(".")[0], allowed) - elif isinstance(node, ast.Call) and isinstance( - node.func, ast.Name - ): - self.assertNotIn(node.func.id, {"open", "__import__"}) - - def test_loaders_have_one_owner(self): - """Workflows use input loaders, not numerical geometry or raw IO.""" - for name in ( - "load_lookup_table", - "_ref_stack_group", - "find_reference_file", - "load_reference_stack", - "default_psf", - "load_psf", - ): - with self.subTest(name=name): - function = getattr(inputs, name) - self.assertEqual(function.__module__, inputs.__name__) - self.assertFalse(hasattr(geometry, name)) - self.assertFalse(hasattr(slap2, name)) - self.assertIs(workflow.inputs, inputs) - self.assertIs(annotation.inputs, inputs) - self.assertIs(inputs.read_tiff, read_tiff) - self.assertIs(inputs.build_combined_psf, geometry.build_combined_psf) - self.assertIs( - inputs.threshold_and_crop_psf, geometry.threshold_and_crop_psf - ) - - def test_reader_protocol_method_is_docstring_only(self): - """The reader contract has no executable placeholder statement.""" - tree = ast.parse(inspect.getsource(slap2.Slap2Reader)) - method = next( - node - for node in tree.body[0].body - if isinstance(node, ast.FunctionDef) and node.name == "getLineData" - ) - self.assertEqual(len(method.body), 1) - self.assertIsNotNone(ast.get_docstring(method)) - self.assertIsNone(method.returns) - - -class TestBandInputLoading(unittest.TestCase): - """Preserve lookup layouts, reference page order and bundled PSF values.""" - - def test_lookup_aliases_layouts_and_int32_columns(self): - """Path groups take precedence; DMD groups and MATLAB layout work.""" - expected = { - "allSuperPixelIDs": np.array([[7], [3], [11]], dtype=np.int32), - "sparseMaskInds": np.array( - [[19, 1], [5, 2], [27, 3]], dtype=np.int32 - ), - "fastZ2RefZ": np.array([[4], [2]], dtype=np.int32), - } - with tempfile.TemporaryDirectory() as directory: - path = Path(directory) / "lookup.h5" - for layout in (None, 0, 1): - with self.subTest(row_major=layout): - with h5py.File(path, "w") as file: - if layout is not None: - file.create_dataset("row_major", data=layout) - for group in ("Path1", "DMD2"): - for key, value in expected.items(): - file.create_dataset( - f"{group}/{key}", - data=(value if layout else value.T).astype( - np.float64 - ), - ) - file.create_group("DMD1") - result = inputs.load_lookup_table(path, 2) - self.assertEqual(set(result), set(expected)) - for key, value in expected.items(): - self.assertEqual(set(result[key]), {"DMD1", "DMD2"}) - for actual in result[key].values(): - self.assertEqual(actual.dtype, np.dtype("int32")) - np.testing.assert_array_equal(actual, value) - - def test_reference_tiff_values_shapes_and_dtypes(self): - """Decoding is raw; band scaling and channel reshape stay local.""" - with tempfile.TemporaryDirectory() as directory: - path = Path(directory) / "sample_DMD2-REFERENCE.tif" - for dtype in (np.uint16, np.float32): - for group in ("Path2", "DMD2"): - with self.subTest(dtype=dtype, group=group): - raw = np.arange(120).reshape(6, 4, 5).astype(dtype) - if dtype == np.float32: - raw[1, 2, 3] = np.nan - tifffile.imwrite(path, raw, photometric="minisblack") - channels = np.array([[2], [1]], dtype=np.int16) - meta = {group: {"channels": channels}} - with patch.object( - inputs, "read_tiff", wraps=read_tiff - ) as reader: - stack, actual_channels, filename = ( - inputs.load_reference_stack(directory, meta, 1) - ) - reader.assert_called_once_with(str(path)) - decoded = read_tiff(path) - self.assertEqual(decoded.dtype, raw.dtype) - np.testing.assert_array_equal(decoded, raw) - expected = (raw / 100).reshape(3, 2, 4, 5) - expected = expected.transpose(1, 0, 2, 3) - self.assertEqual(stack.shape, (2, 3, 4, 5)) - self.assertEqual(stack.dtype, expected.dtype) - np.testing.assert_array_equal(stack, expected) - np.testing.assert_array_equal(actual_channels, [2, 1]) - self.assertEqual(actual_channels.dtype, channels.dtype) - np.testing.assert_array_equal(channels, [[2], [1]]) - self.assertEqual(filename, str(path)) - - def test_reference_discovery_precedence_and_missing(self): - """Recursive CONFIG2 wins; alternate and missing cases still work.""" - with tempfile.TemporaryDirectory() as directory: - root = Path(directory) - fallback = root / "sample_DMD1-REFERENCE.tif" - preferred = root / "nested" / "sample_DMD1_CONFIG2-REFERENCE.tif" - preferred.parent.mkdir() - fallback.touch() - self.assertEqual( - inputs.find_reference_file(root, 0), str(fallback) - ) - preferred.touch() - self.assertEqual( - inputs.find_reference_file(root, 0), str(preferred) - ) - channels = np.array([[3, 1]], dtype=np.uint8) - with patch.object(inputs, "read_tiff") as reader: - stack, actual, filename = inputs.load_reference_stack( - root, {"DMD2": {"channels": channels}}, 1 - ) - reader.assert_not_called() - self.assertIsNone(stack) - self.assertIsNone(filename) - self.assertEqual(actual.dtype, channels.dtype) - np.testing.assert_array_equal(actual, [3, 1]) - - def test_reference_metadata_prefers_path_group(self): - """Retain subgroup identity and reject missing reference metadata.""" - primary = {"channels": [2, 1]} - fallback = {"channels": [3]} - meta = {"Path1": primary, "DMD1": fallback} - self.assertIs(inputs._ref_stack_group(meta, 0), primary) - self.assertIs(inputs._ref_stack_group({"DMD1": fallback}, 0), fallback) - with self.assertRaisesRegex( - ValueError, "Missing reference-stack group Path2 or DMD2" - ): - inputs._ref_stack_group(meta, 1) - - def test_bundled_psf_preserves_asset_values_and_per_dmd_cropping(self): - """Preserve the resource path, float32 conversion and crop.""" - resource = resources.files("giant_python").joinpath( - "assets", "psfs", "dil-17.tif" - ) - with resources.as_file(resource) as path: - original = tifffile.imread(str(path)).astype(np.float32) - with patch.object(inputs, "read_tiff", wraps=read_tiff) as reader: - template = inputs.default_psf(17) - self.assertEqual(Path(reader.call_args.args[0]).name, "dil-17.tif") - reader.assert_called_once() - self.assertEqual(template.dtype, np.dtype("float32")) - np.testing.assert_array_equal(template, original) - expected = original.copy() - expected[expected < expected.max() * np.exp(-3)] = 0 - rows = np.flatnonzero(np.any(expected != 0, axis=1)) - cols = np.flatnonzero(np.any(expected != 0, axis=0)) - row_slice = slice(rows[0], rows[-1] + 1) - col_slice = slice(cols[0], cols[-1] + 1) - expected = expected[row_slice, col_slice] - with patch.object( - inputs, "default_psf", wraps=inputs.default_psf - ) as loader: - result = inputs.load_psf(17, 2) - self.assertEqual(loader.call_args_list, [call(17), call(17)]) - self.assertEqual(set(result), {"DMD1", "DMD2"}) - for actual in result.values(): - self.assertEqual(actual.dtype, np.dtype("float32")) - np.testing.assert_array_equal(actual, expected) - self.assertFalse(np.shares_memory(result["DMD1"], result["DMD2"])) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_band_workflow.py b/tests/test_band_workflow.py deleted file mode 100644 index 9fe88ca..0000000 --- a/tests/test_band_workflow.py +++ /dev/null @@ -1,587 +0,0 @@ -"""Public band orchestration without vendor data or an expensive NMF fit. - -The real driver, per-path composition, low-res assembly, motion geometry, -high-res trace solver, result assembly and HDF5 writer run together. Only -external input reads, background/noise estimation and localization are mocked. -""" - -import unittest -from contextlib import ExitStack -from dataclasses import asdict -from pathlib import Path -from tempfile import TemporaryDirectory -from unittest.mock import patch - -import numpy as np -import torch - -from giant_python.extraction.band import traces -from giant_python.extraction.band import workflow as wf -from giant_python.extraction.band.transient import TransientOptions -from giant_python.extraction.band.types import TrialTraceResult -from giant_python.io.experiment_summary import read_summary -from giant_python.models.params import ( - AnnotationOptions, - BandSiloParams, - ExecutionOptions, -) -from giant_python.models.trial_table import Slap2Info, TrialTable -from giant_python.numerics.baseline import assemble_dff -from giant_python.pipeline.extract import extract_band_sources - - -class TestBandWorkflow(unittest.TestCase): - """Exercise the public API through the driver to saved results.""" - - def test_default_trace_workflow_and_saved_parameters(self): - """Without mode flags, both real solvers feed assembly and HDF5.""" - self._run_mock_session() - - def _run_mock_session(self, *, loaded=False, rois=False): - """Check schema, policy and motion values in a two-path session.""" - n_frames, side, n_sp = 120, 15, 49 - positions = [(r, c) for r in range(4, 11) for c in range(4, 11)] - smi = np.array( - [ - [r * side + c, index + 1] - for index, (r, c) in enumerate(positions) - ], - dtype=np.int32, - ) - frames = np.arange(n_frames, dtype=float) + 11 - psf = np.ones((3, 3), dtype=np.float32) - options = ExecutionOptions(max_workers=3, max_trials=2) - annotation = AnnotationOptions(enabled=rois, interactive=False) - params = BandSiloParams( - analyze_hz=10, denoise_window_s=0.3, baseline_window_s=0.5, - z_tol=2.5, - decay_tau_s=.2, trace_beta_fraction=.001, - trace_max_iter=1500, - ) - before = (asdict(params), asdict(options), asdict(annotation)) - - with TemporaryDirectory() as directory, ExitStack() as stack: - root = Path(directory) - # Deliberately not savedr/trial_table.h5, and not even on disk. - source = root / "selection" / "only-selected-trials.h5" - table = TrialTable( - source_path=source, - savedr=root / "results", - datadr=root / "raw", - filename=np.array( - [ - [ - f"p{d}-skip.dat", - f"p{d}-selected.dat", - f"p{d}-beyond-limit.dat", - ] - for d in range(2) - ], - dtype=object, - ), - slap2_info=Slap2Info( - first_line=np.full((2, 3), 11), - last_line=np.full((2, 3), 130), - ref_stack={"selection": "unchanged"}, - ), - motion_correction={ - "fn_adata": np.array( - [ - [ - f"p{d}-skip.h5", - f"p{d}-selected.h5", - f"p{d}-beyond-limit.h5", - ] - for d in range(2) - ], - dtype=object, - ) - }, - ) - filenames_before = table.filename.copy() - align_before = table.motion_correction["fn_adata"].copy() - lookup = { - "fastZ2RefZ": { - f"DMD{d + 1}": np.array([[10 + d]]) for d in range(2) - }, - "allSuperPixelIDs": { - f"DMD{d + 1}": np.arange(n_sp).reshape(-1, 1) - for d in range(2) - }, - "sparseMaskInds": {f"DMD{d + 1}": smi for d in range(2)}, - } - read_table = stack.enter_context( - patch.object(TrialTable, "from_h5", return_value=table) - ) - prepare = stack.enter_context( - patch.object(wf, "load_trial_table", wraps=wf.load_trial_table) - ) - stack.enter_context( - patch.object( - wf.inputs, "load_lookup_table", return_value=lookup - ) - ) - stack.enter_context( - patch.object( - wf.inputs, - "load_psf", - return_value={"DMD1": psf, "DMD2": psf}, - ) - ) - stack.enter_context( - patch.object( - wf.inputs, - "load_reference_stack", - return_value=(np.zeros((1, 1, side, side)), None, None), - ) - ) - stack.enter_context( - patch.object( - wf.geo, "build_subsample_matrix_inds", return_value=smi - ) - ) - keep = stack.enter_context( - patch.object( - wf, - "compute_keep_trials", - return_value=np.array( - [[False, True, True], [False, True, True]] - ), - ) - ) - stack.enter_context( - patch.object( - wf, - "read_align_info", - return_value=({"DMD1": 10.0, "DMD2": 10.0}, 2), - ) - ) - geometry = stack.enter_context( - patch.object( - wf, "build_user_roi_geometry", return_value=({}, {}) - ) - ) - roi_selection = { - "user_roi_masks": { - f"DMD{d + 1}": [np.ones((1, side, side), dtype=bool)] - for d in range(2) - }, - "user_roi_labels": { - f"DMD{d + 1}": [f"ROI {d}"] for d in range(2) - }, - "user_roi_superpixels": { - f"DMD{d + 1}": [np.array([0, 1])] for d in range(2) - }, - } - resolve_rois = stack.enter_context( - patch.object( - wf, "resolve_user_rois", return_value=roi_selection - ) - ) - - def read_trial( - trial_ix, - keep_trial, - dmd_ix, - samp_freq, - sp_ids, - datadr, - runtime_table, - num_channels, - all_channels=True, - ): - """Supply kept data and verify resolved read inputs.""" - self.assertIs(runtime_table["trial_table"], table) - self.assertEqual(runtime_table["source_path"], source) - self.assertEqual(num_channels, 2) - self.assertEqual(samp_freq, 10) - self.assertEqual(datadr, str(table.datadr)) - self.assertIn(trial_ix, (0, 1)) - if not keep_trial: - return None - self.assertEqual(trial_ix, 1) - self.assertEqual( - runtime_table["filename"][dmd_ix, trial_ix], - f"p{dmd_ix}-selected.dat", - ) - # Internal reference offsets, not saved displacement. - alignment = { - "DSframes": frames, - "motionDSr": np.full(n_frames, 2.0), - "motionDSc": np.full(n_frames, -1.0), - "motionDSz": np.zeros(n_frames), - "onlineYshift": np.full(n_frames, 4.0), - "onlineXshift": np.full(n_frames, 5.0), - "onlineZshift": np.full(n_frames, 6.0), - } - data = np.full( - (n_sp, n_frames), 20 + 10 * dmd_ix, dtype=np.float32 - ) - return { - "data": data, - "data_count": np.full_like(data, 2), - "data2": data * 2, - "data_count2": np.full_like(data, 2), - "alignment": alignment, - "ds_frames": frames, - } - - low_read = stack.enter_context( - patch.object( - wf.td, "read_band_trial_data", side_effect=read_trial - ) - ) - high_read = stack.enter_context( - patch.object( - traces, "read_band_trial_data", side_effect=read_trial - ) - ) - mapper = stack.enter_context( - patch.object( - wf, - "map_trials", - side_effect=lambda fn, items, max_workers, **kw: [ - fn(i) for i in items - ], - ) - ) - stack.enter_context( - patch.object( - wf, - "_estimate_background", - side_effect=lambda data, *args, **kw: data * 0.75, - ) - ) - stack.enter_context( - patch.object( - wf.bg, - "fit_noise_variance_model", - side_effect=lambda data, *args: (np.ones_like(data), 1, 0), - ) - ) - localize = stack.enter_context( - patch.object( - wf, - "_localize", - return_value=( - np.ones((1, side, side), dtype=np.float32), - np.array([[0, 7, 7]]), - torch.ones((side * side, 1)), - torch.tensor( - [[0, 7, 7, 1, 1, 1]], dtype=torch.float32 - ), - np.array([2], dtype=np.float32), - ), - ) - ) - assemble = stack.enter_context( - patch.object( - wf, "assemble_path_summary", wraps=wf.assemble_path_summary - ) - ) - select_bins = stack.enter_context( - patch.object( - wf.motion_binning, "select_motion_bins", - wraps=wf.motion_binning.select_motion_bins, - ) - ) - writer = stack.enter_context( - patch.object( - wf, - "write_summary", - wraps=wf.write_summary, - ) - ) - - bin_trial_motion = stack.enter_context( - patch.object( - traces, "_bin_trial_motion", wraps=traces._bin_trial_motion - ) - ) - - fit_results = [] - original_fit = traces.fit_transient_traces - - def fit_trial(*args, **kwargs): - """Retain the real fit payload to verify independent assembly.""" - result = original_fit(*args, **kwargs) - fit_results.append(result) - return result - - transient_fit = stack.enter_context( - patch.object(traces, "fit_transient_traces", side_effect=fit_trial) - ) - trace_calls = stack.enter_context( - patch.object(wf, "get_high_res_traces", wraps=wf.get_high_res_traces) - ) - stack.enter_context(patch.object( - traces, "_weighted_trace_average", - side_effect=AssertionError("Production must use QR, not simple averages"), - )) - input_value = table if loaded else source - summary = extract_band_sources( - input=input_value, - params=params, - execution=options, - annotations=annotation, - ) - self.assertIs(prepare.call_args.args[0], input_value) - prepare.assert_called_once() - if loaded: - read_table.assert_not_called() - else: - read_table.assert_called_once_with(source) - np.testing.assert_array_equal(table.filename, filenames_before) - np.testing.assert_array_equal( - table.motion_correction["fn_adata"], align_before - ) - self.assertEqual(table.source_path, source) - self.assertEqual( - keep.call_args.args[0][0, 1], - str(table.savedr / "motion_correction" / "p0-selected.h5"), - ) - self.assertEqual(low_read.call_count, 4) - for dmd_ix, call in enumerate(localize.call_args_list): - mask = call.kwargs["exclusion_mask"] - if rois: - np.testing.assert_array_equal( - mask, roi_selection["user_roi_masks"][f"DMD{dmd_ix + 1}"][0] - ) - else: - self.assertIsNone(mask) - self.assertEqual(select_bins.call_count, 2) - for call in select_bins.call_args_list: - self.assertEqual(call.kwargs["z_tol"], params.z_tol) - self.assertEqual(high_read.call_count, 2) - self.assertEqual(transient_fit.call_count, 2) - self.assertEqual(trace_calls.call_count, 4) - for call in trace_calls.call_args_list: - self.assertNotIn("simple_trace_extraction", call.kwargs) - self.assertNotIn("convolution_trace_extraction", call.kwargs) - self.assertIsInstance(call.kwargs["transient_options"], TransientOptions) - for call in transient_fit.call_args_list: - self.assertEqual(call.kwargs["options"].tau_s, params.decay_tau_s) - self.assertEqual(call.kwargs["options"].beta_fraction, params.trace_beta_fraction) - self.assertEqual(call.kwargs["options"].max_iter, params.trace_max_iter) - self.assertEqual(call.kwargs["options"].tolerance, params.trace_tolerance) - self.assertEqual(call.kwargs["options"].debias, params.trace_debias) - self.assertEqual(call.kwargs["options"].event_threshold, params.trace_event_threshold) - self.assertEqual(call.args[4], params.analyze_hz) - self.assertEqual(bin_trial_motion.call_count, 2) - for call in bin_trial_motion.call_args_list: - self.assertEqual(call.kwargs["z_tol"], params.z_tol) - self.assertEqual(mapper.call_count, 4) - for call in mapper.call_args_list: - self.assertEqual(call.args[2], options.max_workers) - self.assertEqual(list(call.args[1]), [0, 1]) - if rois: - geometry.assert_called_once() - self.assertFalse(resolve_rois.call_args.kwargs["interactive"]) - else: - geometry.assert_not_called() - resolve_rois.assert_not_called() - - writer.assert_called_once() - output = ( - table.savedr / "source_extraction" / "experiment_summary.h5" - ) - self.assertIs(writer.call_args.args[0], summary) - self.assertEqual(writer.call_args.args[1], output) - restored = read_summary(output) - self.assertEqual(len(summary.paths), 2) - self.assertEqual(len(restored.paths), 2) - assert summary.params is not None - assert restored.params is not None - self.assertEqual(summary.params["numChannels"], 2) - for name in ("simple_trace_extraction", "convolution_trace_extraction"): - self.assertFalse(hasattr(params, name)) - self.assertNotIn(name, summary.params) - self.assertNotIn(name, restored.params) - self.assertEqual(restored.params["decayTau_s"], .2) - self.assertNotIn("trace_tau_s", restored.params) - self.assertEqual(restored.params["trace_beta_fraction"], .001) - self.assertEqual(restored.params["trace_max_iter"], 1500) - self.assertNotIn("operator", summary.params) - self.assertNotIn("operator", restored.params) - self.assertEqual(summary.params["draw_user_rois"], rois) - for dmd_ix, (path, saved) in enumerate( - zip(summary.paths, restored.paths) - ): - runtime_result = assemble.call_args_list[dmd_ix].args[0] - self.assertEqual(len(runtime_result.trace_results), 1) - self.assertIsInstance( - runtime_result.trace_results[0], TrialTraceResult - ) - trial = runtime_result.trace_results[0] - self.assertEqual(len(trial), 11) - fit = fit_results[dmd_ix] - np.testing.assert_allclose(trial.d_f_denoised, fit.d_f, rtol=1e-6) - np.testing.assert_allclose(trial.events, fit.events, rtol=1e-6) - np.testing.assert_array_equal(trial.events[0], 0) - # Independently check the original QR coefficients, rather - # than comparing two aliases of the convolution result. - fit_args = transient_fit.call_args_list[dmd_ix].args - residual, designs, bins, valid = fit_args[:4] - self.assertTrue(valid.all()) - for motion, design in designs.items(): - selected = bins == motion - expected_ls = np.linalg.lstsq( - design, residual[:, selected], rcond=None - )[0].T - norm = np.sum(design ** 2, axis=0) - expected_f0 = (3 * residual[:, selected]).T @ design / norm - expected_res = ( - residual[:, selected] - design @ expected_ls.T - ).T @ design / norm - np.testing.assert_allclose(trial.d_f[selected], expected_ls, rtol=1e-5) - np.testing.assert_allclose(trial.f0_ls[selected], expected_f0, rtol=1e-5) - np.testing.assert_allclose( - trial.f_residual[selected], expected_res, atol=1e-4 - ) - self.assertEqual(path.path_index, dmd_ix) - np.testing.assert_array_equal(path.z_depths, [[10 + dmd_ix]]) - np.testing.assert_array_equal( - path.frame_info.trial_num_frames, [[n_frames]] - ) - np.testing.assert_array_equal( - path.frame_info.frame_line_idxs.ravel(), frames - ) - np.testing.assert_array_equal( - path.frame_info.offlineYshifts, -2 - ) - np.testing.assert_array_equal( - path.frame_info.offlineXshifts, 1 - ) - np.testing.assert_array_equal(path.frame_info.onlineYshifts, 4) - np.testing.assert_array_equal(path.frame_info.onlineXshifts, 5) - np.testing.assert_allclose( - path.global_f[0], n_sp * (10 + 5 * dmd_ix) - ) - np.testing.assert_array_equal(path.global_f, saved.global_f) - np.testing.assert_array_equal( - path.frame_info.offlineXshifts, - saved.frame_info.offlineXshifts, - ) - np.testing.assert_allclose( - path.sources[0].df_ls, - saved.sources[0].df_ls, - equal_nan=True, - ) - _, expected_f0, expected_ls, _ = assemble_dff( - trial.d_f, trial.f0_ls + trial.f_residual, - runtime_result.denoise_window, runtime_result.baseline_window, - ) - for source in (path.sources[0], saved.sources[0]): - for field, expected_trace in ( - ("df_ls", expected_ls), ("f0", expected_f0), - ("df_denoised", trial.d_f_denoised), - ("events", trial.events), - ): - values = getattr(source, field) - self.assertEqual(values.shape, (2, n_frames)) - np.testing.assert_array_equal(values[0], expected_trace[:, 0]) - self.assertTrue(np.isnan(values[1]).all()) - self.assertFalse(np.allclose( - source.df_ls, source.df_denoised, equal_nan=True - )) - self.assertEqual(len(path.user_rois), int(rois)) - self.assertEqual(saved.annotation_enabled, rois) - if rois: - self.assertEqual(path.user_rois[0].label, f"ROI {dmd_ix}") - self.assertEqual(path.user_rois[0].f.shape, (2, n_frames)) - self.assertNotEqual( - summary.paths[0].global_f[0, 0], - summary.paths[1].global_f[0, 0], - ) - self.assertEqual( - before, (asdict(params), asdict(options), asdict(annotation)) - ) - self.assertFalse(hasattr(params, "num_channels")) - - def test_exact_nonstandard_selected_trial_table_path(self): - """Preserve the chosen filename through extraction and persistence.""" - self._run_mock_session() - - def test_loaded_table_is_not_reopened(self): - """Prepare a loaded model directly without a second file read.""" - self._run_mock_session(loaded=True) - - def test_default_production_uses_qr_and_convolution(self): - """Production uses QR and fits convolution traces without opt-in.""" - with patch.object( - traces, "solve_phi_motion", wraps=traces.solve_phi_motion - ) as qr: - self._run_mock_session(loaded=True) - self.assertGreater(qr.call_count, 0) - - def test_split_policy_preserves_rois_and_caller_options(self): - """Split policy leaves scientific options unchanged.""" - self._run_mock_session(loaded=True, rois=True) - - def test_missing_channel_inference_fails_before_processing(self): - """Missing channel metadata raises before ROI resolution or writing.""" - params = BandSiloParams() - with TemporaryDirectory() as directory, ExitStack() as stack: - table = dict( - savedr=directory, - n_dmds=1, - fn_adata=[], - filename=[], - datadr=directory, - ) - stack.enter_context( - patch.object(wf, "load_trial_table", return_value=table) - ) - stack.enter_context(patch.object(wf.inputs, "load_lookup_table")) - stack.enter_context(patch.object(wf.inputs, "load_psf")) - stack.enter_context(patch.object(wf, "compute_keep_trials")) - stack.enter_context( - patch.object(wf, "read_align_info", return_value=({}, None)) - ) - rois = stack.enter_context( - patch.object(wf, "_resolve_session_user_rois") - ) - process = stack.enter_context(patch.object(wf, "_process_dmd")) - writer = stack.enter_context(patch.object(wf, "write_summary")) - with self.assertRaisesRegex( - ValueError, "num_channels.*provide acquisition metadata" - ): - wf.extract_band_sources("selected.h5", params) - rois.assert_not_called() - process.assert_not_called() - writer.assert_not_called() - self.assertFalse(hasattr(params, "num_channels")) - - def test_enabled_empty_annotations_keep_resolution_context(self): - """An enabled empty selection retains geometry and headless policy.""" - table, lookup = {"n_dmds": 2}, {"lookup": "unchanged"} - geometry, references, selection = {}, {"DMD1": "reference.tif"}, {} - with ( - patch.object( - wf, - "build_user_roi_geometry", - return_value=(geometry, references), - ) as build, - patch.object( - wf, "resolve_user_rois", return_value=selection - ) as resolve, - ): - result = wf._resolve_session_user_rois( - "results", - table, - lookup, - AnnotationOptions(enabled=True, interactive=False), - ) - build.assert_called_once_with(table, lookup) - resolve.assert_called_once_with( - Path("results") / "annotations", - 2, - geometry, - interactive=False, - ref_files=references, - ) - self.assertIs(result, selection) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_bandsilo_annotate.py b/tests/test_bandsilo_annotate.py deleted file mode 100644 index 67828d7..0000000 --- a/tests/test_bandsilo_annotate.py +++ /dev/null @@ -1,227 +0,0 @@ -"""Tests for band annotation's pure seams and split-policy orchestration.""" - -import os -import tempfile -import unittest -from copy import deepcopy -from unittest import mock - -import numpy as np - -from giant_python.extraction.band import annotation as an -from giant_python.io.annotations import save_annotations_h5 -from giant_python.models import ( - AnnotationOptions, - BandSiloParams, - ExecutionOptions, - TrialTable, -) - - -class TestFirstValidTrial(unittest.TestCase): - """first_valid_trial returns the first kept trial or raises.""" - - def test_first_kept(self): - """Returns the index of the first True entry for the DMD.""" - keep = np.array([[False, True, True], [True, False, False]]) - self.assertEqual(an.first_valid_trial(keep, 0), 1) - self.assertEqual(an.first_valid_trial(keep, 1), 0) - - def test_none_kept_raises(self): - """A DMD with no kept trials raises ValueError.""" - keep = np.array([[False, False]]) - with self.assertRaises(ValueError): - an.first_valid_trial(keep, 0) - - -class TestMotionMedian(unittest.TestCase): - """motion_median_from_adata returns per-axis rounded medians.""" - - def test_medians(self): - """Each axis is rounded then reduced by the median.""" - a_data = { - "motionDSr": np.array([1.2, 1.8, 3.1]), - "motionDSc": np.array([0.0, 0.0, 4.0]), - "motionDSz": np.array([5.0, 5.0, 5.0]), - } - self.assertEqual(an.motion_median_from_adata(a_data), (2, 0, 5)) - - -class TestResolveInteractivity(unittest.TestCase): - """resolve_interactivity honors override, env, then TTY fallback.""" - - def test_override_wins(self): - """An explicit override short-circuits detection.""" - self.assertTrue(an.resolve_interactivity(True)) - self.assertFalse(an.resolve_interactivity(False)) - - def test_headless_env(self): - """GIANT_HEADLESS forces headless when no override is given.""" - import os - from unittest import mock - - with mock.patch.dict(os.environ, {"GIANT_HEADLESS": "1"}): - self.assertFalse(an.resolve_interactivity(None)) - - def test_tty_fallback(self): - """With no override/env, falls back to the stdin-TTY probe.""" - import os - from unittest import mock - - with mock.patch.dict(os.environ, {}, clear=True): - with mock.patch.object(an, "_stdin_is_tty", return_value=True): - self.assertTrue(an.resolve_interactivity(None)) - with mock.patch.object(an, "_stdin_is_tty", return_value=False): - self.assertFalse(an.resolve_interactivity(None)) - - -class TestEmptyUserRois(unittest.TestCase): - """_empty_user_rois builds a neutral per-DMD selection.""" - - def test_shape(self): - """Every per-DMD field is an empty list and annotated is False.""" - out = an._empty_user_rois(2) - self.assertFalse(out["annotated"]) - for key in ("DMD1", "DMD2"): - self.assertEqual(out["user_roi_masks"][key], []) - self.assertEqual(out["user_roi_superpixels"][key], []) - self.assertEqual(out["user_roi_labels"][key], []) - self.assertEqual(out["roi_records"][key], []) - - -class TestResolveUserRois(unittest.TestCase): - """resolve_user_rois loads an existing file or fails fast when headless.""" - - def _geo(self): - """Return a minimal single-DMD geometry dict.""" - return { - "DMD1": { - "num_fast_z": 2, - "yx_shape": (4, 5), - "sp_fastz": np.array([0]), - "sp_rows": np.array([1]), - "sp_cols": np.array([1]), - } - } - - def _write_annotations(self, dr): - """Write a one-ROI annotations.h5 into ``dr`` and return its path.""" - mask = np.zeros((2, 4, 5), dtype=bool) - mask[0, 1, 1] = True - records = { - "DMD1": [ - { - "type": "polygon", - "label": "user_roi_1", - "position": np.array( - [[1, 1], [1, 3], [3, 3], [3, 1]], dtype=np.float64 - ), - } - ] - } - return save_annotations_h5(dr, records, {"DMD1": [mask]}, 1) - - def test_loads_existing(self): - """An existing annotations.h5 is loaded even when non-interactive.""" - with tempfile.TemporaryDirectory() as d: - self._write_annotations(d) - out = an.resolve_user_rois(d, 1, self._geo(), interactive=False) - self.assertTrue(out["annotated"]) - self.assertEqual(out["user_roi_labels"]["DMD1"], ["user_roi_1"]) - self.assertTrue(out["user_roi_masks"]["DMD1"][0][0, 1, 1]) - np.testing.assert_array_equal( - out["user_roi_superpixels"]["DMD1"][0], np.array([0]) - ) - - def test_missing_headless_raises(self): - """A missing file + headless raises with actionable guidance.""" - with tempfile.TemporaryDirectory() as d: - with self.assertRaises(RuntimeError) as ctx: - an.resolve_user_rois(d, 1, self._geo(), interactive=False) - self.assertIn("giant annotate", str(ctx.exception)) - - def test_missing_interactive_draws(self): - """A missing file + interactive delegates to the drawing helper.""" - from unittest import mock - - sentinel = {"annotated": True} - with tempfile.TemporaryDirectory() as d: - with mock.patch.object( - an, "_draw_and_save_user_rois", return_value=sentinel - ) as draw: - out = an.resolve_user_rois( - d, 1, self._geo(), interactive=True, ref_files={} - ) - draw.assert_called_once() - self.assertIs(out, sentinel) - - -class TestAnnotationService(unittest.TestCase): - """Annotation validates scoped options before IO without legacy bridges.""" - - def test_split_options_and_in_memory_input(self): - """The explicit step uses GUI/run policy and never mutates callers.""" - input = TrialTable() - params = BandSiloParams() - execution = ExecutionOptions(verbose=True) - annotations = AnnotationOptions(interactive=False) - before = deepcopy((params, execution, annotations)) - table = { - "fn_adata": np.array([["a.h5"]]), - "filename": np.array([["trial"]]), - "datadr": "/data", - "annotation_save_dr": "/annotations", - "moco_save_dr": "/motion", - "n_dmds": 1, - } - with ( - mock.patch.object( - an, "load_trial_table", return_value=table - ) as load, - mock.patch.object( - an, "compute_keep_trials", return_value=np.array([[True]]) - ), - mock.patch.object(an.inputs, "load_lookup_table"), - mock.patch.object( - an, "build_user_roi_geometry", return_value=({}, {}) - ), - mock.patch.object(an, "resolve_user_rois") as resolve, - mock.patch.object(an, "log") as log, - ): - result = an.annotate_band_rois( - input, - params=params, - execution=execution, - annotations=annotations, - ) - load.assert_called_once_with(input) - resolve.assert_called_once_with( - "/annotations", 1, {}, interactive=False, ref_files={} - ) - self.assertEqual( - result, os.path.join("/annotations", "annotations.h5") - ) - self.assertTrue(all(call.args[1] for call in log.call_args_list)) - self.assertEqual((params, execution, annotations), before) - self.assertFalse(annotations.enabled) - - def test_mixed_options_rejected_before_io(self): - """Mis-scoped dictionaries and removed keywords cannot load data.""" - with mock.patch.object(an, "load_trial_table") as load: - for kwargs in ( - {"params": {"max_workers": 2}}, - {"params": {"operator": "Ada"}}, - {"params": {"scan_mode": "band"}}, - {"execution": {"enabled": True}}, - {"annotations": {"draw_user_rois": True}}, - {"params_in": {}}, - ): - with self.subTest(kwargs=kwargs): - with self.assertRaises(TypeError): - an.annotate_band_rois("trial.h5", **kwargs) - load.assert_not_called() - self.assertFalse(hasattr(an, "_resolve_params")) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_bandsilo_background.py b/tests/test_bandsilo_background.py deleted file mode 100644 index ff078c5..0000000 --- a/tests/test_bandsilo_background.py +++ /dev/null @@ -1,612 +0,0 @@ -"""Tests for canonical band background, activity, and motion kernels. - -Covers the motion-binning, PSF/convolution, interpolated-background, -noise-model, and rho kernels with synthetic fixtures. ``build_interp_data`` is -additionally cross-checked against a verbatim copy of the reference in the -project's development notes; here we assert its structural behavior. The rho -path is exercised end-to-end on a small synthetic geometry built from the real -:mod:`giant_python.extraction.band.geometry` helpers, checked for linearity in -residual (rho is a linear projection of the residual). -""" - -import unittest - -import numpy as np -import torch - -from giant_python.extraction.band import activity -from giant_python.extraction.band import background as bg -from giant_python.extraction.band import geometry as geo -from giant_python.extraction.band import motion_binning, operators - - -def _small_geometry(seed=0, positions_per_plane=None): - """Build a small consistent DMD geometry for rho/convolution tests. - - Returns a dict of the geometry arrays, PSF tensors, selected pixels, per - plane convolution matrices, and per-motion H matrices. - - ``positions_per_plane`` is a list (one entry per z-plane) of ``(row, col)`` - reference-pixel positions. It defaults to a dense 7x7 block in every plane - so the wide horizontal erosion in compute_rho leaves a non-empty valid - region. - """ - rng = np.random.default_rng(seed) - npc = 15 # dmd_pixels_per_column - npr = 15 # dmd_pixels_per_row - plane_size = npc * npr - - if positions_per_plane is None: - block = [(r, c) for r in range(4, 11) for c in range(4, 11)] - positions_per_plane = [block, block] - num_fast_zs = len(positions_per_plane) - - rows = [] - for z, positions in enumerate(positions_per_plane): - for r, c in positions: - rows.append([z * plane_size + r * npr + c, len(rows) + 1]) - subsample_matrix_inds = np.array(rows, dtype=np.int32) - num_super_pixels = subsample_matrix_inds.shape[0] - - # A small normalized 5x5 PSF (odd so it has a well-defined center). - yy, xx = np.mgrid[-2:3, -2:3] - psf2d = np.exp(-(yy**2 + xx**2) / 2.0).astype(np.float32) - - ref_d, ref_c, ref_r = geo.ref_pixs_to_drc( - subsample_matrix_inds[:, 0], npc, npr - ) - sparse_h_inds, sparse_h_vals = operators.build_sparse_h( - subsample_matrix_inds, psf2d, npc, npr - ) - - umyx = np.array([[0, 0], [1, 0]], dtype=np.float64) - sel_pix_mask, sel_pix_idxs = geo.build_selected_pixel_mask( - umyx, ref_d, ref_r, ref_c, num_fast_zs, npc, npr, psf2d - ) - - psf_tensor, psf_center, psf_tensor_exp, psf_center_exp = ( - operators.expand_psf(psf2d) - ) - - d_mats = [] - d_mats_exp = [] - for z in range(num_fast_zs): - _, sel_2d = geo.selected_pixels_2d_for_plane(sel_pix_idxs, z, npc, npr) - d_mats.append( - operators.build_convolution_matrix(sel_2d, psf_tensor, psf_center) - ) - d_mats_exp.append( - operators.build_convolution_matrix( - sel_2d, psf_tensor_exp, psf_center_exp - ) - ) - - h_mots = operators.build_motion_h_matrices( - sparse_h_inds, sparse_h_vals, umyx, sel_pix_idxs, num_super_pixels, npr - ) - - return dict( - rng=rng, - npc=npc, - npr=npr, - num_fast_zs=num_fast_zs, - num_super_pixels=num_super_pixels, - psf2d=psf2d, - ref_d=ref_d, - ref_r=ref_r, - ref_c=ref_c, - umyx=umyx, - sel_pix_idxs=sel_pix_idxs, - d_mats=d_mats, - d_mats_exp=d_mats_exp, - h_mots=h_mots, - ) - - -class TestMotionBinning(unittest.TestCase): - """bin_motion / select_motion_bins / bin_motion_yx.""" - - def test_bin_motion_groups_identical_rounded_vectors(self): - """Frames with equal rounded motion share a bin index.""" - mr = np.array([0.1, 0.2, 5.0]) - mc = np.array([0.0, 0.0, 0.0]) - mz = np.array([0.0, 0.0, 0.0]) - unique_motion, mot_inds = motion_binning.bin_motion(mr, mc, mz) - self.assertEqual(unique_motion.shape[1], 3) - self.assertEqual(mot_inds[0], mot_inds[1]) - self.assertNotEqual(mot_inds[0], mot_inds[2]) - self.assertEqual(mot_inds.ndim, 1) - - def test_select_motion_bins_filters_z_and_count(self): - """Bins far in z or with too few frames are dropped.""" - # bin 0: z=0, many frames; bin 1: z=0, few frames; bin 2: z=10, many - unique_motion = np.array( - [[0, 0, 0], [1, 0, 0], [2, 0, 10]], dtype=float - ) - mot_inds = np.array([0] * 5 + [1] * 2 + [2] * 5) - motion_z = np.zeros(12) - keep, frames = motion_binning.select_motion_bins( - unique_motion, mot_inds, motion_z, z_tol=1.5, min_frames=3 - ) - self.assertEqual(list(keep), [0]) - self.assertEqual(int(frames.sum()), 5) - - def test_z_tolerance_changes_selected_bins(self): - """Tolerance includes its boundary without relaxing frame counts.""" - unique_motion = np.array([[0, 0, 0], [0, 0, 2], [1, 0, 0]]) - mot_inds = np.array([0] * 202 + [1] * 101 + [2] * 100) - motion_z = unique_motion[mot_inds, 2] - for tolerance, expected in ((1.5, [0]), (2.0, [0, 1])): - with self.subTest(tolerance=tolerance): - keep, frames = motion_binning.select_motion_bins( - unique_motion, mot_inds, motion_z, z_tol=tolerance - ) - np.testing.assert_array_equal(keep, expected) - np.testing.assert_array_equal(frames, np.isin(mot_inds, expected)) - - def test_bin_motion_yx_marks_dropped_frames(self): - """Frames outside frames_to_keep get index -1.""" - mr = np.array([0.0, 0.0, 1.0, 1.0]) - mc = np.array([0.0, 0.0, 0.0, 0.0]) - frames_to_keep = np.array([True, True, True, False]) - umyx, mot_inds_yx = motion_binning.bin_motion_yx( - mr, mc, frames_to_keep - ) - self.assertEqual(mot_inds_yx[3], -1) - self.assertEqual(mot_inds_yx[0], mot_inds_yx[1]) - self.assertEqual(umyx.shape[1], 2) - - -class TestSelectedPixels(unittest.TestCase): - """build_selected_pixel_mask / pixel_coords / selected_pixels_2d.""" - - def test_mask_and_coords_round_trip(self): - """Flat selected indices decode back to their z/row/col.""" - ref_d = np.array([0]) - ref_r = np.array([5]) - ref_c = np.array([5]) - umyx = np.array([[0, 0]], dtype=float) - psf2d = np.ones((3, 3), dtype=np.float32) - mask, idxs = geo.build_selected_pixel_mask( - umyx, ref_d, ref_r, ref_c, 1, 12, 12, psf2d - ) - self.assertEqual(int(mask.sum()), idxs.size) - coords = geo.pixel_coords_from_idxs(idxs, 12, 12) - # every decoded coordinate should be marked in the mask - for z, r, c in coords: - self.assertTrue(mask[z, r, c]) - - def test_selected_pixels_2d_for_plane(self): - """Plane selection returns the in-plane row/col for that z only.""" - idxs = np.array([0 * 25 + 2 * 5 + 3, 1 * 25 + 4 * 5 + 1]) - z0, sel0 = geo.selected_pixels_2d_for_plane(idxs, 0, 5, 5) - self.assertEqual(sel0.tolist(), [[2, 3]]) - z1, sel1 = geo.selected_pixels_2d_for_plane(idxs, 1, 5, 5) - self.assertEqual(sel1.tolist(), [[4, 1]]) - - -class TestPsfConvolution(unittest.TestCase): - """expand_psf and build_convolution_matrix.""" - - def test_expand_psf_normalized_and_larger(self): - """Both PSFs sum to 1 and the expanded one is ex_fac larger.""" - yy, xx = np.mgrid[-2:3, -2:3] - psf2d = np.exp(-(yy**2 + xx**2) / 2.0).astype(np.float32) - pt, pc, pte, pce = operators.expand_psf(psf2d, ex_fac=2) - self.assertAlmostEqual(float(pt.sum()), 1.0, places=5) - self.assertAlmostEqual(float(pte.sum()), 1.0, places=5) - self.assertEqual(tuple(pte.shape), (10, 10)) - self.assertEqual(pc, (2, 2)) - self.assertEqual(pce, (5, 5)) - - def test_convolution_matrix_values(self): - """D[t, s] equals the PSF weight at the target-source offset.""" - yy, xx = np.mgrid[-2:3, -2:3] - psf2d = np.exp(-(yy**2 + xx**2) / 2.0).astype(np.float32) - pt, pc, _, _ = operators.expand_psf(psf2d) - sel_2d = np.array([[5, 5], [6, 5], [5, 8]]) - d = operators.build_convolution_matrix(sel_2d, pt, pc) - # self entry == center weight - self.assertAlmostEqual(float(d[0, 0]), float(pt[pc[0], pc[1]]), 5) - # offset (1, 0): target row+1 -> psf[center+1, center] - self.assertAlmostEqual( - float(d[1, 0]), float(pt[pc[0] + 1, pc[1]]), places=5 - ) - # offset (0, 3): rel_col = center + 3 = 5 is outside the 5x5 PSF -> 0 - self.assertEqual(float(d[2, 0]), 0.0) - - def test_convolution_matrix_empty(self): - """An empty plane yields a 0x0 matrix.""" - pt = torch.ones((3, 3)) - d = operators.build_convolution_matrix( - np.zeros((0, 2), dtype=int), pt, (1, 1) - ) - self.assertEqual(tuple(d.shape), (0, 0)) - - -class TestInterpData(unittest.TestCase): - """build_interp_data structural behavior and edge branches.""" - - def test_basic_fill_and_bounds(self): - """Interpolation fills selected pixels within the motion window.""" - ref_r = np.array([4, 6, 8], dtype=np.int32) - ref_c = np.array([5, 5, 5], dtype=np.int32) - data = np.array([[1.0], [3.0], [5.0]], dtype=np.float32) # one frame - sel_2d = np.array([[4, 5], [5, 5], [6, 5], [7, 5]]) - umyx = np.array([[0, 0]], dtype=float) - mot_yx = np.array([0]) - out, rb, cb = bg.build_interp_data( - data, ref_r, ref_c, sel_2d, umyx, mot_yx - ) - # exact rows (4, 6) match data; row 5 interpolates to 2.0 - self.assertAlmostEqual(float(out[0, 0]), 1.0, places=5) - self.assertAlmostEqual(float(out[1, 0]), 2.0, places=5) - self.assertAlmostEqual(float(out[2, 0]), 3.0, places=5) - self.assertEqual(cb, (5, 5)) - - def test_cubic_interpolation(self): - """Natural cubic interpolation is smooth and keeps exact samples.""" - ref_r = np.array([0, 2, 4, 6], dtype=np.int32) - ref_c = np.full(4, 5, dtype=np.int32) - data = np.array([[0.0], [1.0], [0.0], [1.0]], dtype=np.float32) - sel_2d = np.array([[0, 5], [1, 5], [2, 5], [3, 5]]) - umyx = np.array([[0, 0]], dtype=float) - mot_yx = np.array([0]) - linear, _, _ = bg.build_interp_data( - data, ref_r, ref_c, sel_2d, umyx, mot_yx, method="linear" - ) - cubic, _, _ = bg.build_interp_data( - data, ref_r, ref_c, sel_2d, umyx, mot_yx, method="cubic" - ) - self.assertAlmostEqual(float(cubic[0, 0]), 0.0, places=5) - self.assertAlmostEqual(float(cubic[2, 0]), 1.0, places=5) - self.assertNotAlmostEqual( - float(cubic[1, 0]), float(linear[1, 0]), places=5 - ) - - def test_cubic_falls_back_with_two_samples(self): - """Two-point columns retain the linear result in cubic mode.""" - ref_r = np.array([4, 6], dtype=np.int32) - ref_c = np.array([5, 5], dtype=np.int32) - data = np.array([[1.0], [3.0]], dtype=np.float32) - sel_2d = np.array([[5, 5]]) - umyx = np.array([[0, 0]], dtype=float) - mot_yx = np.array([0]) - out, _, _ = bg.build_interp_data( - data, ref_r, ref_c, sel_2d, umyx, mot_yx, method="cubic" - ) - self.assertAlmostEqual(float(out[0, 0]), 2.0, places=5) - - def test_unknown_interpolation_method_rejected(self): - """Interpolation method typos fail before processing data.""" - with self.assertRaisesRegex(ValueError, "linear.*cubic"): - bg.build_interp_data( - np.ones((1, 1)), - np.array([1]), - np.array([1]), - np.array([[1, 1]]), - np.array([[0, 0]]), - np.array([0]), - method="spline", - ) - - def test_empty_motion_and_missing_column(self): - """Empty bins and columns absent from the selection are skipped.""" - ref_r = np.array([4, 6], dtype=np.int32) - ref_c = np.array([3, 3], dtype=np.int32) # column 3 not selected - data = np.array([[1.0], [2.0]], dtype=np.float32) - sel_2d = np.array([[4, 5], [6, 5]]) # only column 5 selected - umyx = np.array([[0, 0], [1, 0]], dtype=float) - mot_yx = np.array([0]) # bin 1 has no frames - out, _, _ = bg.build_interp_data( - data, ref_r, ref_c, sel_2d, umyx, mot_yx - ) - # column 3 not in selection -> nothing filled - self.assertTrue(np.all(np.isnan(out))) - - def test_single_point_column(self): - """A single-reference-pixel column is copied, not interpolated.""" - ref_r = np.array([5], dtype=np.int32) - ref_c = np.array([5], dtype=np.int32) - data = np.array([[7.0]], dtype=np.float32) - sel_2d = np.array([[5, 5]]) - umyx = np.array([[0, 0]], dtype=float) - mot_yx = np.array([0]) - out, _, _ = bg.build_interp_data( - data, ref_r, ref_c, sel_2d, umyx, mot_yx - ) - self.assertAlmostEqual(float(out[0, 0]), 7.0, places=5) - - def test_all_exact_column_skips_interpolation(self): - """A multi-point column whose targets all match exactly interpolates - nothing (the interp branch is empty).""" - ref_r = np.array([4, 6], dtype=np.int32) - ref_c = np.array([5, 5], dtype=np.int32) - data = np.array([[1.0], [3.0]], dtype=np.float32) - # selected rows are exactly the reference rows -> all exact matches - sel_2d = np.array([[4, 5], [6, 5]]) - umyx = np.array([[0, 0]], dtype=float) - mot_yx = np.array([0]) - out, _, _ = bg.build_interp_data( - data, ref_r, ref_c, sel_2d, umyx, mot_yx - ) - self.assertAlmostEqual(float(out[0, 0]), 1.0, places=5) - self.assertAlmostEqual(float(out[1, 0]), 3.0, places=5) - - def test_out_of_window_motion(self): - """A motion shift pushing all refs outside the window fills nothing.""" - ref_r = np.array([4, 6], dtype=np.int32) - ref_c = np.array([5, 5], dtype=np.int32) - data = np.array([[1.0], [2.0]], dtype=np.float32) - sel_2d = np.array([[4, 5], [6, 5]]) - # single bin, but with a huge column shift beyond width via tiny W - umyx = np.array([[0, 0]], dtype=float) - mot_yx = np.array([0]) - out, _, _ = bg.build_interp_data( - data, ref_r, ref_c, sel_2d, umyx, mot_yx, height=1, width=1 - ) - self.assertTrue(np.all(np.isnan(out))) - - -class TestRollingBaseline(unittest.TestCase): - """baseline_window_frames + compute_rolling_baseline.""" - - def test_window_frames(self): - """Window length is int(hz * seconds).""" - self.assertEqual(bg.baseline_window_frames(80.0, 4.0), 320) - - def test_no_nan_matches_brute_median(self): - """With no NaNs, the baseline is a centered moving median.""" - - def brute_rolling_median(data, window): - """Compute clipped centered medians from finite row samples.""" - n = data.shape[1] - out = np.full(data.shape, np.nan, dtype=float) - for r in range(data.shape[0]): - for i in range(n): - lo = max(0, i - (window - 1) // 2) - hi = min(n, i + window // 2 + 1) - vals = data[r, lo:hi] - vals = vals[~np.isnan(vals)] - if vals.size: - out[r, i] = np.median(vals) - return out - - data = np.arange(20, dtype=np.float32).reshape(2, 10) - for window in (3, 5): - expected = brute_rolling_median(data, window) - out = bg.compute_rolling_baseline(data.copy(), window) - np.testing.assert_allclose(out, expected, rtol=1e-5) - - def test_ignores_nan_in_window(self): - """NaNs are dropped from each window's median.""" - data = np.array([[1.0, np.nan, 3.0, 100.0, 5.0]], dtype=np.float32) - out = bg.compute_rolling_baseline(data.copy(), 3) - # window at idx 1 sees {1, 3} -> median 2; idx 2 sees {3, 100} -> 51.5 - np.testing.assert_allclose(out[0, 0], 1.0) - np.testing.assert_allclose(out[0, 1], 2.0) - np.testing.assert_allclose(out[0, 2], 51.5) - - def test_all_nan_row_stays_nan(self): - """A fully-NaN row yields a fully-NaN baseline.""" - data = np.full((1, 6), np.nan, dtype=np.float32) - out = bg.compute_rolling_baseline(data, 3) - self.assertTrue(np.all(np.isnan(out))) - - -class TestBackgroundAssembly(unittest.TestCase): - """assemble_background maps selected-pixel background to superpixels.""" - - def test_maps_reference_pixels(self): - """Each superpixel receives its shifted reference-pixel background.""" - npc = npr = 10 - plane = npc * npr - ref_d = np.array([0, 0]) - ref_r = np.array([2, 4]) - ref_c = np.array([3, 5]) - # sel pixels are exactly the (unshifted) reference pixels - sel_pix_idxs = np.sort( - np.array([0 * plane + 2 * npr + 3, 0 * plane + 4 * npr + 5]) - ) - interp_bg = np.array([[1.0, 1.0], [2.0, 2.0]], dtype=np.float32) - umyx = np.array([[0, 0]], dtype=float) - mot_yx = np.array([0, 0]) - out = bg.assemble_background( - interp_bg, - umyx, - mot_yx, - sel_pix_idxs, - ref_d, - ref_r, - ref_c, - n_super_pixels=2, - n_frames=2, - dmd_pixels_per_column=npc, - dmd_pixels_per_row=npr, - ) - self.assertFalse(np.any(np.isnan(out))) - self.assertEqual(out.shape, (2, 2)) - - -class TestNoiseModel(unittest.TestCase): - """fit_noise_variance_model + compute_residual.""" - - def test_data_std_positive_and_residual_zscored(self): - """data_std is positive; residual is (data-bg)/std.""" - rng = np.random.default_rng(1) - # Per-pixel mean/count spread so var_pred varies (else the bright - # selection is empty and the fit degenerates). - means = np.linspace(1.0, 10.0, 30).astype(np.float32) - background = np.tile(means[:, None], (1, 200)) - low = background + rng.standard_normal((30, 200)).astype(np.float32) - v_im = np.tile( - (0.05 + 0.01 * np.arange(30)).astype(np.float32)[:, None], - (1, 200), - ) - data_std, v_k, v_b = bg.fit_noise_variance_model( - low, background, v_im, vif=1.38 - ) - self.assertTrue(np.all(data_std > 0)) - self.assertTrue(np.isfinite(v_b)) - residual = bg.compute_residual(low, background, data_std) - np.testing.assert_allclose( - residual, (low - background) / data_std, rtol=1e-6 - ) - - -class TestRho(unittest.TestCase): - """build_motion_h_matrices + compute_rho + smoothing.""" - - def test_h_matrices_shape(self): - """One sparse H per motion bin, sized (superpixels, sel pixels).""" - g = _small_geometry() - self.assertEqual(len(g["h_mots"]), len(g["umyx"])) - h0 = g["h_mots"][0] - self.assertEqual( - tuple(h0.size()), - (g["num_super_pixels"], g["sel_pix_idxs"].shape[0]), - ) - - def test_compute_rho_shape_and_linearity(self): - """rho has the selected-pixel shape and is linear in the residual.""" - g = _small_geometry() - n_frames = 12 - n_sp = g["num_super_pixels"] - residual = ( - g["rng"].standard_normal((n_sp, n_frames)).astype(np.float32) - ) - # assign frames to the two motion bins - mot_yx = np.array([0, 1] * (n_frames // 2), dtype=np.int32) - - kwargs = dict( - unique_motion_to_keep_yx=g["umyx"], - h_mots=g["h_mots"], - d_mats=g["d_mats"], - d_mats_expanded=g["d_mats_exp"], - sel_pix_idxs=g["sel_pix_idxs"], - ref_d=g["ref_d"], - ref_r=g["ref_r"], - ref_c=g["ref_c"], - num_fast_zs=g["num_fast_zs"], - dmd_pixels_per_column=g["npc"], - dmd_pixels_per_row=g["npr"], - psf2d=g["psf2d"], - ) - rho = activity.compute_rho(residual, mot_yx, **kwargs) - self.assertEqual(rho.shape, (g["sel_pix_idxs"].shape[0], n_frames)) - self.assertTrue(np.any(np.isfinite(rho))) - - rho2 = activity.compute_rho(2.0 * residual, mot_yx, **kwargs) - finite = np.isfinite(rho) & np.isfinite(rho2) - np.testing.assert_allclose( - rho2[finite], 2.0 * rho[finite], rtol=1e-4, atol=1e-5 - ) - - def test_compute_rho_handles_empty_motion_and_empty_plane(self): - """A frameless motion bin and an empty z-plane are skipped.""" - g = _small_geometry() - n_frames = 6 - residual = ( - g["rng"] - .standard_normal((g["num_super_pixels"], n_frames)) - .astype(np.float32) - ) - # All frames in bin 0; bin 1 gets none -> motion_frames empty branch - mot_yx = np.zeros(n_frames, dtype=np.int32) - # Add a third, always-empty z-plane -> new_ncols == 0 branch - rho = activity.compute_rho( - residual, - mot_yx, - unique_motion_to_keep_yx=g["umyx"], - h_mots=g["h_mots"], - d_mats=g["d_mats"] + [torch.zeros((0, 0))], - d_mats_expanded=g["d_mats_exp"] + [torch.zeros((0, 0))], - sel_pix_idxs=g["sel_pix_idxs"], - ref_d=g["ref_d"], - ref_r=g["ref_r"], - ref_c=g["ref_c"], - num_fast_zs=g["num_fast_zs"] + 1, - dmd_pixels_per_column=g["npc"], - dmd_pixels_per_row=g["npr"], - psf2d=g["psf2d"], - ) - self.assertEqual(rho.shape[1], n_frames) - - def test_compute_rho_skips_plane_with_no_valid_columns(self): - """A plane whose eroded valid region is empty is skipped (its rho - rows stay NaN) while the dense plane is still filled.""" - block = [(r, c) for r in range(4, 11) for c in range(4, 11)] - # plane 1 has a single reference pixel -> erosion wipes its valid - # region even though it has selected pixels and H columns. - g = _small_geometry(positions_per_plane=[block, [(7, 7)]]) - n_frames = 6 - residual = ( - g["rng"] - .standard_normal((g["num_super_pixels"], n_frames)) - .astype(np.float32) - ) - mot_yx = np.array([0, 1] * (n_frames // 2), dtype=np.int32) - rho = activity.compute_rho( - residual, - mot_yx, - unique_motion_to_keep_yx=g["umyx"], - h_mots=g["h_mots"], - d_mats=g["d_mats"], - d_mats_expanded=g["d_mats_exp"], - sel_pix_idxs=g["sel_pix_idxs"], - ref_d=g["ref_d"], - ref_r=g["ref_r"], - ref_c=g["ref_c"], - num_fast_zs=g["num_fast_zs"], - dmd_pixels_per_column=g["npc"], - dmd_pixels_per_row=g["npr"], - psf2d=g["psf2d"], - ) - plane_size = g["npc"] * g["npr"] - z_of_sel = g["sel_pix_idxs"] // plane_size - # plane 0 produced some finite rho; plane 1 (single pixel) did not - self.assertTrue(np.any(np.isfinite(rho[z_of_sel == 0]))) - self.assertTrue(np.all(np.isnan(rho[z_of_sel == 1]))) - - def test_mask_high_nan_rho(self): - """Rows that are mostly NaN are fully NaN'd; fraction is returned.""" - rho = np.array( - [[1.0, 2.0, 3.0, 4.0], [np.nan, np.nan, np.nan, 1.0]], - dtype=np.float32, - ) - nan_ct = activity.mask_high_nan_rho(rho, thresh=0.5) - self.assertAlmostEqual(nan_ct[0], 0.0) - self.assertAlmostEqual(nan_ct[1], 0.75) - self.assertTrue(np.all(np.isnan(rho[1]))) - self.assertFalse(np.any(np.isnan(rho[0]))) - - def test_decay_kernel_normalized(self): - """The decay kernel sums to 1 and rises to its last sample.""" - k = activity.decay_kernel_1d(0.15, 80.0) - self.assertAlmostEqual(float(k.sum()), 1.0, places=6) - self.assertEqual(np.argmax(k), len(k) - 1) - - def test_smooth_rho_preserves_constant(self): - """A finite constant row is preserved; an all-NaN row stays NaN.""" - rho = np.vstack( - [ - np.full((1, 50), 2.0, dtype=np.float32), - np.full((1, 50), np.nan, dtype=np.float32), - ] - ) - k = activity.decay_kernel_1d(0.05, 80.0) - out = activity.smooth_rho(rho, k) - # interior samples of the constant row remain ~2.0 - np.testing.assert_allclose(out[0, 10:40], 2.0, rtol=1e-4) - self.assertTrue(np.all(np.isnan(out[1]))) - - def test_smooth_rho_all_nan_chunk(self): - """A chunk with no finite data is passed through unchanged.""" - rho = np.full((3, 20), np.nan, dtype=np.float32) - k = activity.decay_kernel_1d(0.05, 80.0) - out = activity.smooth_rho(rho, k) - self.assertTrue(np.all(np.isnan(out))) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_bandsilo_baseline.py b/tests/test_bandsilo_baseline.py deleted file mode 100644 index 27fcf83..0000000 --- a/tests/test_bandsilo_baseline.py +++ /dev/null @@ -1,130 +0,0 @@ -"""Tests for giant_python.numerics.baseline (Phase 7 F0 / dF-F). - -Covers the NaN-aware moving mean, the rolling-min-envelope ``compute_f0`` -baseline estimator, and the dF/F assembly. ``compute_f0`` is additionally -cross-checked against a verbatim copy of the reference in the project's -development notes; here we assert structural properties on synthetic traces. -""" - -import unittest - -import numpy as np - -from giant_python.numerics import baseline as bl - - -class TestMovmeanNan(unittest.TestCase): - """_movmean_nan edge cases.""" - - def test_window_one_is_identity(self): - """A window of 1 returns a float copy of the input.""" - x = np.array([1.0, np.nan, 3.0]) - out = bl._movmean_nan(x, 1) - np.testing.assert_array_equal( - np.nan_to_num(out), np.nan_to_num(x.astype(float)) - ) - - def test_ignores_nan(self): - """NaNs are excluded from the mean, not treated as zero.""" - x = np.array([2.0, np.nan, 4.0]) - out = bl._movmean_nan(x, 3) - # center window over index 1 sees {2, 4} -> mean 3 - self.assertAlmostEqual(out[1], 3.0) - - def test_all_nan_yields_nan(self): - """A window with no valid samples yields NaN (den == 0 branch).""" - x = np.full(5, np.nan) - out = bl._movmean_nan(x, 3) - self.assertTrue(np.all(np.isnan(out))) - - -class TestComputeF0(unittest.TestCase): - """compute_f0 shape handling and baseline behavior.""" - - def _drifting_trace(self, total=200, seed=0): - """A drifting baseline plus positive transients and noise.""" - rng = np.random.default_rng(seed) - t = np.arange(total) - base = 5 + 2 * np.sin(t / 40) - transients = np.clip(rng.standard_normal(total), 0, None) * 3 - return (base + transients + rng.standard_normal(total) * 0.2).astype( - float - ) - - def test_tracks_below_signal(self): - """F0 stays at or below the (transient-carrying) signal.""" - f = self._drifting_trace() - f0 = bl.compute_f0(f, 9, 40) - self.assertEqual(f0.shape, f.shape) - # baseline should sit near the lower envelope, well below the mean - self.assertLess(np.nanmean(f0), np.nanmean(f)) - - def test_two_dimensional(self): - """A 2-D input is handled column-wise and keeps its shape.""" - f = np.stack( - [self._drifting_trace(seed=1), self._drifting_trace(seed=2)], - axis=1, - ) - f[10:15, 0] = np.nan - f0 = bl.compute_f0(f, 9, 40) - self.assertEqual(f0.shape, f.shape) - - def test_all_nan_column_skipped(self): - """An all-NaN column is returned unchanged (all NaN).""" - f = np.stack( - [self._drifting_trace(seed=3), np.full(200, np.nan)], axis=1 - ) - f0 = bl.compute_f0(f, 9, 40) - self.assertTrue(np.all(np.isnan(f0[:, 1]))) - self.assertFalse(np.all(np.isnan(f0[:, 0]))) - - def test_short_input_returns_mean(self): - """Fewer than 4 samples returns the broadcast column mean.""" - f = np.array([[2.0, 8.0], [4.0, 10.0], [6.0, 12.0]]) - f0 = bl.compute_f0(f, 9, 40) - np.testing.assert_allclose(f0[:, 0], 4.0) - np.testing.assert_allclose(f0[:, 1], 10.0) - - def test_sparse_hull_triggers_gap_interpolation(self): - """A trace with NaN gaps exercises the residual-NaN interpolation.""" - f = self._drifting_trace(total=120, seed=4) - f[30:70] = np.nan # a wide gap forces envelope NaNs to be filled - f0 = bl.compute_f0(f, 15, 80) - self.assertEqual(f0.shape, f.shape) - self.assertFalse(np.all(np.isnan(f0))) - - def test_copy_on_write_readonly_rolling_result(self): - """Under pandas copy-on-write, to_numpy() is read-only; f0 still fits. - - Regression: ``rolling().median().to_numpy()`` returns a read-only view - when copy-on-write is enabled (the default in newer pandas), which used - to break the in-place per-column baseline assignment. - """ - import pandas as pd - - f = np.stack( - [self._drifting_trace(seed=1), self._drifting_trace(seed=2)], - axis=1, - ) - with pd.option_context("mode.copy_on_write", True): - f0 = bl.compute_f0(f, 9, 40) - self.assertEqual(f0.shape, f.shape) - self.assertFalse(np.all(np.isnan(f0))) - - -class TestAssembleDff(unittest.TestCase): - """assemble_dff combines the least-squares traces into dF/F.""" - - def test_relationships(self): - """F = dF_ls + F0_ls; dF = F - F0; dFF = dF / clip(F0).""" - rng = np.random.default_rng(0) - d_f_ls = np.clip(rng.standard_normal((150, 2)), 0, None) * 2 - f0_ls = np.full((150, 2), 5.0) + rng.standard_normal((150, 2)) * 0.1 - f, f0, d_f, d_ff = bl.assemble_dff(d_f_ls, f0_ls, 9, 40) - np.testing.assert_allclose(f, d_f_ls + f0_ls) - np.testing.assert_allclose(d_f, f - f0) - np.testing.assert_allclose(d_ff, d_f / np.clip(f0, 1e-4, None)) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_bandsilo_geometry.py b/tests/test_bandsilo_geometry.py deleted file mode 100644 index c7587dd..0000000 --- a/tests/test_bandsilo_geometry.py +++ /dev/null @@ -1,179 +0,0 @@ -"""Tests for canonical band geometry, input loading, and sparse operators.""" - -import os -import tempfile -import unittest - -import h5py -import numpy as np -import tifffile - -from giant_python.extraction.band.geometry import ( - _pad_to, - build_combined_psf, - build_subsample_matrix_inds, - ref_pixs_to_drc, - threshold_and_crop_psf, -) -from giant_python.extraction.band.inputs import ( - default_psf, - find_reference_file, - load_lookup_table, - load_psf, - load_reference_stack, -) -from giant_python.extraction.band.operators import build_sparse_h - - -class TestRefPixsToDrc(unittest.TestCase): - """ref_pixs_to_drc maps flat pixels to (depth, column, row).""" - - def test_known_mapping(self): - """A hand-computed pixel decomposes to the expected d/c/r.""" - # C (per column) = 4, R (per row) = 5 -> plane = 20. - # pix = d*plane + c*C + r = 1*20 + 2*4 + 2 = 30 - d, c, r = ref_pixs_to_drc(np.array([30]), 4, 5) - self.assertEqual((int(d[0]), int(c[0]), int(r[0])), (1, 2, 2)) - - -class TestSubsampleMatrixInds(unittest.TestCase): - """Preserve the original middle-by-input-order representative pixel.""" - - def test_median_reference_pixel(self): - """Sorted pixels select their middle entry for odd and even lengths.""" - all_ids = np.array([[1], [2]], dtype=np.int32) - mask = np.array( - [[10, 1], [12, 1], [14, 1], [20, 2], [22, 2]], dtype=np.int32 - ) - out = build_subsample_matrix_inds(all_ids, mask) - np.testing.assert_array_equal( - out, np.array([[11, 1], [21, 2]], dtype=np.int32) - ) - - def test_odd_length_preserves_middle_index(self): - """The structural refactor does not change unsorted pixel selection.""" - all_ids = np.array([[1]], dtype=np.int32) - # The original implementation selects 9, not the sorted median 11. - # Changing that rule would be a separate scientific behavior change. - mask = np.array([[14, 1], [10, 1], [12, 1]], dtype=np.int32) - out = build_subsample_matrix_inds(all_ids, mask) - np.testing.assert_array_equal(out, np.array([[9, 1]], dtype=np.int32)) - - -class TestSparseH(unittest.TestCase): - """build_sparse_h assembles the PSF-convolution COO matrix.""" - - def test_known_values(self): - """Indices/values match a hand-computed single-superpixel case.""" - sub = np.array([[30, 1]], dtype=np.int32) # d=1, c=2, r=2 (C=4,R=5) - psf = np.array( - [[0.0, 1.0, 0.0], [2.0, 3.0, 4.0], [0.0, 5.0, 0.0]], - dtype=np.float32, - ) - inds, vals = build_sparse_h(sub, psf, 4, 5) - np.testing.assert_array_equal(inds[0], np.zeros(5, dtype=np.int32)) - np.testing.assert_array_equal( - inds[1], np.array([27, 31, 32, 33, 37], dtype=np.int32) - ) - np.testing.assert_array_equal( - vals, np.array([1, 2, 3, 4, 5], dtype=np.float32) - ) - - -class TestLookupTable(unittest.TestCase): - """load_lookup_table reshapes per-DMD lookup arrays.""" - - def test_load(self): - """Superpixel ids/mask/fastZ arrays load in expected shapes.""" - with tempfile.TemporaryDirectory() as d: - path = os.path.join(d, "bandRegLookupTable.h5") - with h5py.File(path, "w") as f: - f.create_dataset("row_major", data=1) - p1 = f.create_group("Path1") - p1.create_dataset("allSuperPixelIDs", data=np.array([1, 2, 3])) - p1.create_dataset( - "sparseMaskInds", - data=np.array([[1, 1], [2, 1], [3, 2]]), - ) - p1.create_dataset("fastZ2RefZ", data=np.array([1, 2])) - out = load_lookup_table(path, 1) - self.assertEqual(out["allSuperPixelIDs"]["DMD1"].shape, (3, 1)) - self.assertEqual(out["sparseMaskInds"]["DMD1"].shape, (3, 2)) - self.assertEqual(out["fastZ2RefZ"]["DMD1"].shape, (2, 1)) - - -class TestReferenceFiles(unittest.TestCase): - """find_reference_file and load_reference_stack.""" - - def test_find_first_pattern_and_missing(self): - """CONFIG2 pattern is matched; an empty dir returns None.""" - with tempfile.TemporaryDirectory() as d: - name = "expt_DMD1_CONFIG2-REFERENCE.tif" - open(os.path.join(d, name), "wb").close() - self.assertTrue(find_reference_file(d, 0).endswith(name)) - with tempfile.TemporaryDirectory() as empty: - self.assertIsNone(find_reference_file(empty, 0)) - - def test_load_reference_stack(self): - """A synthetic REFERENCE tif reshapes to [channel, z, y, x].""" - meta = {"Path1": {"channels": np.array([1, 2])}} - with tempfile.TemporaryDirectory() as d: - data = (np.arange(6 * 4 * 4).reshape(6, 4, 4)).astype(np.float32) - tifffile.imwrite( - os.path.join(d, "expt_DMD1-REFERENCE.tif"), - data, - photometric="minisblack", - ) - stack, channels, ref_file = load_reference_stack(d, meta, 0) - self.assertEqual(stack.shape, (2, 3, 4, 4)) - self.assertEqual(list(channels), [1, 2]) - self.assertTrue(ref_file.endswith("expt_DMD1-REFERENCE.tif")) - - def test_load_reference_stack_missing(self): - """No REFERENCE file yields (None, channels, None).""" - meta = {"Path1": {"channels": np.array([1, 2])}} - with tempfile.TemporaryDirectory() as d: - stack, channels, ref_file = load_reference_stack(d, meta, 0) - self.assertIsNone(stack) - self.assertIsNone(ref_file) - self.assertEqual(list(channels), [1, 2]) - - -class TestPsf(unittest.TestCase): - """PSF loading, padding, thresholding, cropping.""" - - def test_default_psf(self): - """A bundled PSF template loads as a 2-D float32 array.""" - psf = default_psf(17) - self.assertEqual(psf.ndim, 2) - self.assertEqual(psf.dtype, np.float32) - - def test_pad_and_combine(self): - """build_combined_psf center-pads to common dims.""" - p1 = np.ones((3, 3), dtype=np.float32) - p2 = np.full((1, 1), 5.0, dtype=np.float32) - combined = build_combined_psf([p1, p2]) - self.assertEqual(combined.shape, (2, 3, 3)) - padded = _pad_to(p2, 3, 3) - self.assertEqual(padded.shape, (3, 3)) - - def test_threshold_and_crop(self): - """Sub-threshold pixels are zeroed and boundary zeros cropped.""" - arr = np.zeros((5, 5), dtype=np.float32) - arr[2, 2] = 10.0 - arr[1, 2] = arr[3, 2] = arr[2, 1] = arr[2, 3] = 1.0 - arr[0, 0] = 0.1 # below 10*exp(-3) ~ 0.498 -> zeroed - cropped = threshold_and_crop_psf(arr) - self.assertEqual(cropped.shape, (3, 3)) - self.assertEqual(cropped[1, 1], 10.0) - - def test_load_psf_from_assets(self): - """load_psf always builds per-DMD PSFs from bundled assets.""" - psf = load_psf(17, 2) - self.assertEqual(set(psf), {"DMD1", "DMD2"}) - self.assertEqual(psf["DMD1"].ndim, 2) - self.assertEqual(psf["DMD1"].dtype, np.float32) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_bandsilo_gui.py b/tests/test_bandsilo_gui.py deleted file mode 100644 index 1ab61e8..0000000 --- a/tests/test_bandsilo_gui.py +++ /dev/null @@ -1,232 +0,0 @@ -"""Tests for canonical band ROI geometry and annotation IO.""" - -import os -import tempfile -import unittest - -import h5py -import numpy as np - -from giant_python.extraction.band.annotation import load_annotations_h5 -from giant_python.extraction.band.rois import ( - _read_roi_records, - compute_user_roi_geometry, - user_roi_superpixel_lists_from_masks, -) -from giant_python.io.annotations import _h5_scalar_str, save_annotations_h5 -from giant_python.io.hdf5 import _read_group - - -def _str_dt(): - """Return the h5py variable-length utf-8 string dtype.""" - return h5py.string_dtype(encoding="utf-8") - - -class TestSuperpixelLists(unittest.TestCase): - """user_roi_superpixel_lists_from_masks selects per-ROI superpixels.""" - - def test_overlapping_rois(self): - """A superpixel inside two ROIs appears in both lists.""" - sp_fastz = np.array([0, 0, 1]) - sp_rows = np.array([1, 2, 0]) - sp_cols = np.array([1, 2, 0]) - mask_a = np.zeros((2, 4, 5), dtype=bool) - mask_a[0, 1, 1] = True - mask_a[0, 2, 2] = True - mask_b = np.zeros((2, 4, 5), dtype=bool) - mask_b[0, 2, 2] = True - out = user_roi_superpixel_lists_from_masks( - [mask_a, mask_b], sp_fastz, sp_rows, sp_cols - ) - np.testing.assert_array_equal(out[0], np.array([0, 1])) - np.testing.assert_array_equal(out[1], np.array([1])) - - -class TestComputeUserRoiGeometry(unittest.TestCase): - """compute_user_roi_geometry derives channel/z-map/superpixel indices.""" - - def _ref(self): - """Return a 2-channel ref stack with channel 1 brighter.""" - ref = np.zeros((2, 3, 4, 5), dtype=np.float32) - ref[1] = 10.0 - return ref - - def test_no_motion(self): - """Zero motion: z_map is fastZ-1; indices from pixel decomposition.""" - ref = self._ref() - fastz_to_refz = np.array([[1], [2], [3]]) - sub = np.array([[22, 1]], dtype=np.int32) - geo = compute_user_roi_geometry(ref, fastz_to_refz, sub, (0, 0, 0)) - self.assertEqual(geo["best_ch"], 1) - self.assertEqual(geo["num_ref_z"], 3) - self.assertEqual(geo["yx_shape"], (4, 5)) - self.assertEqual(geo["num_fast_z"], 3) - np.testing.assert_array_equal(geo["z_map"], np.array([0, 1, 2])) - np.testing.assert_array_equal(geo["sp_fastz"], np.array([1])) - np.testing.assert_array_equal(geo["sp_cols"], np.array([0])) - np.testing.assert_array_equal(geo["sp_rows"], np.array([2])) - self.assertTrue(geo["sp_mask"][1, 2, 0]) - self.assertEqual(geo["sp_mask"].sum(), 1) - - def test_with_motion(self): - """Nonzero motion shifts z_map and row/column indices.""" - ref = self._ref() - fastz_to_refz = np.array([[1], [2], [3]]) - sub = np.array([[22, 1]], dtype=np.int32) - geo = compute_user_roi_geometry(ref, fastz_to_refz, sub, (1, 1, 1)) - np.testing.assert_array_equal(geo["z_map"], np.array([1, 2, 3])) - np.testing.assert_array_equal(geo["sp_rows"], np.array([3])) - np.testing.assert_array_equal(geo["sp_cols"], np.array([1])) - - def test_all_invalid_leaves_mask_empty(self): - """Out-of-bounds superpixels leave sp_mask all False.""" - ref = self._ref() - fastz_to_refz = np.array([[1]]) - sub = np.array([[100, 1]], dtype=np.int32) - geo = compute_user_roi_geometry(ref, fastz_to_refz, sub, (0, 0, 0)) - self.assertEqual(geo["sp_mask"].sum(), 0) - - def test_empty_channels(self): - """A zero-channel ref falls back to best_ch = 0.""" - ref = np.zeros((0, 3, 4, 5), dtype=np.float32) - geo = compute_user_roi_geometry( - ref, np.array([[1]]), np.zeros((0, 2), dtype=np.int32), (0, 0, 0) - ) - self.assertEqual(geo["best_ch"], 0) - - -class TestH5ScalarStr(unittest.TestCase): - """_h5_scalar_str decodes scalar/array/bytes string datasets.""" - - def test_variants(self): - """Bytes and 1-element arrays decode to plain str.""" - with tempfile.TemporaryDirectory() as d: - path = os.path.join(d, "s.h5") - with h5py.File(path, "w") as f: - f.create_dataset("a", data="hi", dtype=_str_dt()) - f.create_dataset( - "b", data=np.array(["x"], dtype=object), dtype=_str_dt() - ) - f.create_dataset("c", data=5) - with h5py.File(path, "r") as f: - self.assertEqual(_h5_scalar_str(f["a"]), "hi") - self.assertEqual(_h5_scalar_str(f["b"]), "x") - self.assertEqual(_h5_scalar_str(f["c"]), "5") - - -class TestAnnotationsRoundTrip(unittest.TestCase): - """save_annotations_h5 / load_annotations_h5 round trip.""" - - def _geo(self): - """Return a minimal single-DMD geometry dict.""" - return { - "DMD1": { - "num_fast_z": 2, - "yx_shape": (4, 5), - "sp_fastz": np.array([0]), - "sp_rows": np.array([1]), - "sp_cols": np.array([1]), - } - } - - def _records_and_masks(self): - """Return one ROI record + matching mask for DMD1.""" - mask = np.zeros((2, 4, 5), dtype=bool) - mask[0, 1, 1] = True - records = { - "DMD1": [ - { - "type": "polygon", - "label": "user_roi_1", - "position": np.array( - [[1, 1], [1, 3], [3, 3], [3, 1]], dtype=np.float64 - ), - } - ] - } - return records, {"DMD1": [mask]} - - def test_round_trip(self): - """A saved ROI reloads with matching label, mask, and superpixels.""" - records, masks = self._records_and_masks() - with tempfile.TemporaryDirectory() as d: - path = save_annotations_h5( - d, records, masks, 1, ref_files={"DMD1": "/data/ref.tif"} - ) - self.assertTrue(os.path.exists(path)) - skip, m, sps, labels, recs = load_annotations_h5( - path, 1, self._geo() - ) - self.assertTrue(skip) - self.assertEqual(labels["DMD1"], ["user_roi_1"]) - self.assertTrue(m["DMD1"][0][0, 1, 1]) - np.testing.assert_array_equal(sps["DMD1"][0], np.array([0])) - self.assertEqual(recs["DMD1"][0]["type"], "polygon") - self.assertIn("position", recs["DMD1"][0]) - - def test_missing_file(self): - """A missing annotations file returns the empty result.""" - out = load_annotations_h5("/nope/annotations.h5", 1, self._geo()) - self.assertEqual(out, (False, {}, {}, {}, {})) - - def test_missing_path_group(self): - """A file with no Path group yields no valid selection.""" - with tempfile.TemporaryDirectory() as d: - path = os.path.join(d, "annotations.h5") - with h5py.File(path, "w") as f: - f["row_major"] = 1 - out = load_annotations_h5(path, 1, self._geo()) - self.assertEqual(out, (False, {}, {}, {}, {})) - - def test_bad_mask_shape(self): - """A mask shape mismatch treats the path as no selection.""" - records, masks = self._records_and_masks() - with tempfile.TemporaryDirectory() as d: - path = save_annotations_h5(d, records, masks, 1) - geo = self._geo() - geo["DMD1"]["num_fast_z"] = 3 - out = load_annotations_h5(path, 1, geo) - self.assertEqual(out, (False, {}, {}, {}, {})) - - def test_empty_records_not_valid(self): - """A path with zero ROIs is read but not marked valid.""" - with tempfile.TemporaryDirectory() as d: - path = save_annotations_h5(d, {"DMD1": []}, {"DMD1": []}, 1) - out = load_annotations_h5(path, 1, self._geo()) - self.assertEqual(out, (False, {}, {}, {}, {})) - - def test_unreadable_file(self): - """A non-HDF5 file that exists returns the empty result.""" - with tempfile.TemporaryDirectory() as d: - path = os.path.join(d, "annotations.h5") - with open(path, "w") as fh: - fh.write("not an hdf5 file") - out = load_annotations_h5(path, 1, self._geo()) - self.assertEqual(out, (False, {}, {}, {}, {})) - - -class TestReadRoiRecords(unittest.TestCase): - """_read_roi_records handles missing masks, labels, and gaps.""" - - def test_defaults_and_missing_mask(self): - """Missing label/type default; missing mask -> zero mask.""" - expected = (2, 4, 5) - with tempfile.TemporaryDirectory() as d: - path = os.path.join(d, "a.h5") - with h5py.File(path, "w") as f: - grp = f.create_group("Path1") - grp.create_dataset("n_rois", data=2) - grp.create_group("roi_000") - with h5py.File(path, "r") as f: - masks, labels, recs, bad = _read_roi_records( - _read_group(f["Path1"], row_major=True), expected - ) - self.assertFalse(bad) - self.assertEqual(labels, ["ROI1"]) - self.assertEqual(recs[0]["type"], "polygon") - self.assertEqual(masks[0].shape, expected) - self.assertFalse(masks[0].any()) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_bandsilo_hdf5.py b/tests/test_bandsilo_hdf5.py deleted file mode 100644 index 9c1acad..0000000 --- a/tests/test_bandsilo_hdf5.py +++ /dev/null @@ -1,308 +0,0 @@ -"""Tests for shared HDF5 IO and band runtime input preparation.""" - -import os -import tempfile -import unittest - -import h5py -import numpy as np - -from giant_python.extraction.band.inputs import ( - _resolve_fn_adata, - compute_keep_trials, - load_alignment_data_h5, - load_trial_table, - read_align_info, -) -from giant_python.io.hdf5 import ( - _decode_one, - _decode_strings, - load_struct_h5, - to_serializable, - write_dict_to_h5group, -) -from giant_python.io.slap2 import _reshape_1d - - -def _str_dt(): - """Return the h5py variable-length utf-8 string dtype.""" - return h5py.string_dtype(encoding="utf-8") - - -class TestToSerializable(unittest.TestCase): - """to_serializable converts numpy structures to plain Python.""" - - def test_dict_list_tuple(self): - """Nested dict/list/tuple are recursed into.""" - out = to_serializable({"a": [np.int64(1), (np.float64(2.0),)]}) - self.assertEqual(out, {"a": [1, (2.0,)]}) - self.assertIsInstance(out["a"][1], tuple) - - def test_ndarray_and_generic(self): - """ndarray -> list and numpy scalar -> Python scalar.""" - self.assertEqual(to_serializable(np.arange(3)), [0, 1, 2]) - self.assertEqual(to_serializable(np.float32(1.5)), 1.5) - - def test_passthrough(self): - """Plain Python values pass through unchanged.""" - self.assertEqual(to_serializable("x"), "x") - - -class TestDecodeHelpers(unittest.TestCase): - """_decode_one / _decode_strings handle bytes, str, arrays.""" - - def test_decode_one(self): - """bytes decode to str; str stays str.""" - self.assertEqual(_decode_one(b"ab"), "ab") - self.assertEqual(_decode_one("cd"), "cd") - - def test_decode_scalar_and_array(self): - """Scalar bytes -> str; object array -> decoded object array.""" - self.assertEqual(_decode_strings(b"z"), "z") - arr = _decode_strings(np.array([b"a", b"b"], dtype=object)) - self.assertEqual(list(arr), ["a", "b"]) - - def test_decode_empty_array(self): - """An empty array is returned unchanged (no vectorize crash).""" - out = _decode_strings(np.array([], dtype=object)) - self.assertEqual(out.size, 0) - - -class TestReadDataset(unittest.TestCase): - """load_struct_h5 read-side orientation and string handling.""" - - def test_scalar_string(self): - """A scalar string dataset reads back as plain str.""" - with tempfile.TemporaryDirectory() as d: - path = os.path.join(d, "s.h5") - with h5py.File(path, "w") as f: - f.create_dataset("datadr", data="hello", dtype=_str_dt()) - out = load_struct_h5(path) - self.assertEqual(out["datadr"], "hello") - - def test_single_element_string_array(self): - """A 1-element string array collapses to a scalar str.""" - with tempfile.TemporaryDirectory() as d: - path = os.path.join(d, "s.h5") - with h5py.File(path, "w") as f: - f.create_dataset( - "name", - data=np.array(["x"], dtype=object), - dtype=_str_dt(), - ) - out = load_struct_h5(path) - self.assertEqual(out["name"], "x") - - def test_string_grid_column_major_transposed(self): - """A 2-D string grid is transposed when column-major (no flag).""" - with tempfile.TemporaryDirectory() as d: - path = os.path.join(d, "s.h5") - grid = np.array([["a", "b", "c"], ["d", "e", "f"]], dtype=object) - with h5py.File(path, "w") as f: - f.create_dataset("filename", data=grid, dtype=_str_dt()) - out = load_struct_h5(path) - self.assertEqual(out["filename"].shape, (3, 2)) - - def test_numeric_transpose_and_row_major(self): - """Numeric 2-D arrays honor the row_major flag.""" - arr = np.arange(6).reshape(2, 3) - with tempfile.TemporaryDirectory() as d: - col = os.path.join(d, "c.h5") - with h5py.File(col, "w") as f: - f.create_dataset("m", data=arr) - self.assertEqual(load_struct_h5(col)["m"].shape, (3, 2)) - - rowm = os.path.join(d, "r.h5") - with h5py.File(rowm, "w") as f: - f.create_dataset("row_major", data=1) - f.create_dataset("m", data=arr) - got = load_struct_h5(rowm)["m"] - self.assertEqual(got.shape, (2, 3)) - np.testing.assert_array_equal(got, arr) - - def test_nested_group(self): - """Nested groups become nested dicts.""" - with tempfile.TemporaryDirectory() as d: - path = os.path.join(d, "s.h5") - with h5py.File(path, "w") as f: - f.create_dataset("row_major", data=1) - g = f.create_group("inner") - g.create_dataset("v", data=np.arange(3)) - out = load_struct_h5(path) - np.testing.assert_array_equal(out["inner"]["v"], np.arange(3)) - - -class TestWriteRoundTrip(unittest.TestCase): - """write_dict_to_h5group covers every value type.""" - - def test_round_trip(self): - """All supported value kinds survive a write/read round trip.""" - payload = { - "row_major": 1, - "flag": True, - "name": "expt", - "count": 7, - "scale": 2.5, - "nums": [1, 2, 3], - "words": ["a", "b"], - "empty": [], - "missing": None, - "weird": 1 + 2j, - "nested": {"x": 1}, - } - with tempfile.TemporaryDirectory() as d: - path = os.path.join(d, "w.h5") - with h5py.File(path, "w") as f: - write_dict_to_h5group(f, payload) - out = load_struct_h5(path) - - self.assertNotIn("empty", out) - self.assertNotIn("missing", out) - self.assertNotIn("row_major", out) - self.assertEqual(out["name"], "expt") - self.assertEqual(int(out["count"]), 7) - self.assertAlmostEqual(float(out["scale"]), 2.5) - self.assertEqual(int(out["flag"]), 1) - np.testing.assert_array_equal(out["nums"], np.array([1, 2, 3])) - self.assertEqual(list(out["words"]), ["a", "b"]) - self.assertEqual(out["weird"], "(1+2j)") - self.assertEqual(int(out["nested"]["x"]), 1) - - -class TestReshapeAndAlignment(unittest.TestCase): - """_reshape_1d and load_alignment_data_h5.""" - - def test_reshape_1d(self): - """Missing key -> None; present key -> flattened array.""" - self.assertIsNone(_reshape_1d({}, "k")) - np.testing.assert_array_equal( - _reshape_1d({"k": np.array([[1], [2]])}, "k"), - np.array([1, 2]), - ) - - def test_load_alignment_data(self): - """aData loads flat arrays; missing online fields become None.""" - with tempfile.TemporaryDirectory() as d: - path = os.path.join(d, "a.h5") - with h5py.File(path, "w") as f: - f.create_dataset("row_major", data=1) - f.create_dataset("DSframes", data=np.arange(4)) - f.create_dataset("motionDSr", data=np.zeros(4)) - f.create_dataset("motionDSc", data=np.zeros(4)) - f.create_dataset("motionDSz", data=np.zeros(4)) - f.create_dataset("numChannels", data=2) - f.create_dataset("alignHz", data=100.0) - out = load_alignment_data_h5(path) - self.assertEqual(out["numChannels"], 2) - self.assertAlmostEqual(out["alignHz"], 100.0) - self.assertEqual(out["DSframes"].shape, (4,)) - self.assertIsNone(out["onlineYshift"]) - - -class TestTrialTable(unittest.TestCase): - """load_trial_table resolve step + keep-trial / align-info helpers.""" - - def test_resolve_fn_adata(self): - """Empty entries stay empty; others become absolute paths.""" - arr = np.array([["", "a.h5"]], dtype=object) - out = _resolve_fn_adata(arr, os.path.join("root", "moco")) - self.assertEqual(out[0, 0], "") - self.assertEqual(out[0, 1], os.path.join("root", "moco", "a.h5")) - - def _write_trial_table(self, path, savedr="/res", with_fn_adata=True): - """Write a minimal synthetic trial_table.h5 (row-major).""" - fnames = np.array([["t1.dat", "t2.dat"]], dtype=object) - fn_adata = np.array([["t1_AD.h5", ""]], dtype=object) - with h5py.File(path, "w") as f: - f.create_dataset("row_major", data=1) - f.create_dataset("datadr", data="/data", dtype=_str_dt()) - f.create_dataset("savedr", data=savedr, dtype=_str_dt()) - f.create_dataset("filename", data=fnames, dtype=_str_dt()) - si = f.create_group("slap2_info") - si.create_dataset("first_line", data=np.array([[1, 2]])) - si.create_dataset("last_line", data=np.array([[10, 20]])) - rs = si.create_group("ref_stack") - p1 = rs.create_group("Path1") - p1.create_dataset("channels", data=np.array([1, 2])) - mc = f.create_group("motion_correction") - if with_fn_adata: - mc.create_dataset("fn_adata", data=fn_adata, dtype=_str_dt()) - ap = mc.create_group("align_params") - ap.create_dataset("align_hz", data=80.0) - - def test_load_trial_table(self): - """Resolved dict is flat with hoisted arrays and absolute paths.""" - with tempfile.TemporaryDirectory() as d: - path = os.path.join(d, "trial_table.h5") - self._write_trial_table(path) - out = load_trial_table(path) - - self.assertEqual( - os.path.normpath(out["datadr"]), os.path.normpath("/data") - ) - self.assertEqual( - os.path.normpath(out["savedr"]), os.path.normpath("/res") - ) - self.assertEqual( - os.path.normpath(out["moco_save_dr"]), - os.path.normpath(os.path.join("/res", "motion_correction")), - ) - self.assertEqual(out["n_dmds"], 1) - self.assertEqual(out["n_trials"], 2) - self.assertAlmostEqual(out["align_params"]["align_hz"], 80.0) - self.assertEqual(out["fn_adata"][0, 1], "") - self.assertEqual( - os.path.normpath(out["fn_adata"][0, 0]), - os.path.normpath( - os.path.join("/res", "motion_correction", "t1_AD.h5") - ), - ) - self.assertIn("Path1", out["ref_stack"]) - np.testing.assert_array_equal(out["first_line"], np.array([[1, 2]])) - - def test_load_trial_table_unregistered(self): - """A table without motion_correction/fn_adata is rejected.""" - with tempfile.TemporaryDirectory() as d: - path = os.path.join(d, "trial_table.h5") - self._write_trial_table(path, with_fn_adata=False) - with self.assertRaises(ValueError): - load_trial_table(path) - - def test_compute_keep_trials(self): - """Only trials whose aData and source files both exist are kept.""" - with tempfile.TemporaryDirectory() as d: - adata = os.path.join(d, "t1_AD.h5") - with open(adata, "w") as fh: - fh.write("x") - with open(os.path.join(d, "t1.dat"), "w") as fh: - fh.write("x") - fn_adata = np.array([[adata, ""]], dtype=object) - filename = np.array([["t1.dat", "t2.dat"]], dtype=object) - keep = compute_keep_trials(fn_adata, filename, d) - np.testing.assert_array_equal(keep, np.array([[True, False]])) - - def test_read_align_info(self): - """align_hz/num_channels come from the first kept trial's aData.""" - with tempfile.TemporaryDirectory() as d: - adata = os.path.join(d, "t1_AD.h5") - with h5py.File(adata, "w") as f: - f.create_dataset("row_major", data=1) - f.create_dataset("numChannels", data=2) - f.create_dataset("alignHz", data=80.0) - fn_adata = np.array([[adata, ""]], dtype=object) - keep = np.array([[True, False]]) - align_hz, num_channels = read_align_info(fn_adata, keep, 1) - self.assertAlmostEqual(align_hz["DMD1"], 80.0) - self.assertEqual(num_channels, 2) - - def test_read_align_info_no_valid(self): - """A DMD with no kept trials contributes no align_hz entry.""" - fn_adata = np.array([["", ""]], dtype=object) - keep = np.array([[False, False]]) - align_hz, num_channels = read_align_info(fn_adata, keep, 1) - self.assertEqual(align_hz, {}) - self.assertIsNone(num_channels) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_bandsilo_nmf.py b/tests/test_bandsilo_nmf.py deleted file mode 100644 index dfe9c60..0000000 --- a/tests/test_bandsilo_nmf.py +++ /dev/null @@ -1,584 +0,0 @@ -"""Tests for canonical band localization (Phase 6 source localization). - -Covers the profile/init kernels, the least-squares/SNR helpers, and the full -``fit_sources`` driver (multiplicative NMF + Adam Gaussian fit + variance -sort + SNR pruning) on a small synthetic geometry. We assert structural -behavior and determinism rather than bit-for-bit agreement with the -reference's regularized normal-equation solve. -""" - -import unittest - -import numpy as np -import torch - -from giant_python.extraction.band import activity, geometry -from giant_python.extraction.band import localization as nmf -from tests.test_bandsilo_background import _small_geometry - - -def _tiny_coords(size=5): - """Return (pixel_coords, sel_pix_idxs, n_pixels) for one square plane.""" - rr, cc = np.meshgrid(np.arange(size), np.arange(size), indexing="ij") - coords = np.column_stack( - [np.zeros(size * size), rr.ravel(), cc.ravel()] - ).astype(np.int32) - sel_pix_idxs = np.arange(size * size) - return coords, sel_pix_idxs, size * size - - -class TestProfiles(unittest.TestCase): - """sel_pix_gaussian_profile / sel_pix_patch_profile.""" - - def test_gaussian_profile_normalized_and_confined(self): - """Each Gaussian column sums to 1 and only its z-plane is populated.""" - coords, _, _ = _tiny_coords(5) - pct = torch.tensor(coords, dtype=torch.float32) - params = torch.tensor([[0.0, 2.0, 2.0, 1.0, 1.0, 0.0]]) - prof = nmf.sel_pix_gaussian_profile(params, pct) - self.assertEqual(prof.shape, (25, 1)) - self.assertAlmostEqual(float(prof.sum()), 1.0, places=5) - # peak at the center pixel (row 2, col 2 -> flat index 12) - self.assertEqual(int(torch.argmax(prof[:, 0])), 12) - - def test_gaussian_profile_wrong_plane_empty(self): - """A source on a different z-plane yields an all-nan column.""" - coords, _, _ = _tiny_coords(5) - pct = torch.tensor(coords, dtype=torch.float32) - params = torch.tensor( - [[1.0, 2.0, 2.0, 1.0, 1.0, 0.0]] - ) # z=1, no pixels - prof = nmf.sel_pix_gaussian_profile(params, pct) - # sum is 0 before normalization -> division yields nan - self.assertTrue(torch.all(torch.isnan(prof))) - - def test_patch_profile_membership(self): - """The patch is the strict box within the radii on the source plane.""" - coords, _, _ = _tiny_coords(5) - pct = torch.tensor(coords, dtype=torch.float32) - params = torch.tensor([[0.0, 2.0, 2.0, 2.0, 2.0]]) # radius 2 - patch = nmf.sel_pix_patch_profile(params, pct) - self.assertEqual(patch.shape, (25, 1)) - # |y-2|<2 and |x-2|<2 -> rows/cols {1,2,3} -> 9 pixels - self.assertEqual(int(patch.sum()), 9) - - -class TestInit(unittest.TestCase): - """init_source_params / build_a_patches / project_spatial_profiles.""" - - def test_init_source_params(self): - """Params start as [z, y, x, 1, 1, 0].""" - seeds = np.array([[0.0, 3.0, 4.0], [1.0, 5.0, 6.0]]) - params = nmf.init_source_params(seeds) - self.assertEqual(tuple(params.shape), (2, 6)) - np.testing.assert_array_equal(params[:, 3:5].numpy(), np.ones((2, 2))) - np.testing.assert_array_equal(params[:, 5].numpy(), np.zeros(2)) - - def test_build_a_patches(self): - """Patches are True only within each source's box on the pixel grid.""" - coords, sel, n_pix = _tiny_coords(5) - pct = torch.tensor(coords, dtype=torch.float32) - seeds = np.array([[0.0, 2.0, 2.0]]) - patches = nmf.build_a_patches(seeds, pct, sel, n_pix, d_xy=2) - self.assertEqual(tuple(patches.shape), (25, 1)) - self.assertEqual(int(patches.sum()), 9) - - def test_project_spatial_profiles_normalized_in_patch(self): - """A is normalized to unit column mass and zero outside the patch.""" - coords, sel, n_pix = _tiny_coords(5) - pct = torch.tensor(coords, dtype=torch.float32) - seeds = np.array([[0.0, 2.0, 2.0]]) - params = nmf.init_source_params(seeds) - patches = nmf.build_a_patches(seeds, pct, sel, n_pix, d_xy=2) - a = nmf.project_spatial_profiles(params, pct, sel, n_pix, patches) - self.assertAlmostEqual(float(a.sum()), 1.0, places=5) - # every nonzero pixel lies inside the patch - self.assertTrue(torch.all(patches[a > 0])) - - -class TestVarianceSortAndReorder(unittest.TestCase): - """variance_sortorder / reorder_sources.""" - - def test_variance_sortorder_descending(self): - """Sources are ordered by descending temporal variance.""" - phi = torch.zeros((10, 3)) - phi[:, 0] = torch.linspace(0, 1, 10) # small variance - phi[:, 1] = torch.linspace(0, 10, 10) # large variance - phi[:, 2] = torch.linspace(0, 5, 10) # medium - order = nmf.variance_sortorder(phi, 3) - self.assertEqual(list(order), [1, 2, 0]) - - def test_reorder_sources(self): - """Reordering reindexes every per-source array consistently.""" - order = np.array([2, 0, 1]) - source_params = torch.arange(18).reshape(3, 6).float() - source_seeds = np.arange(9).reshape(3, 3).astype(float) - a = torch.arange(12).reshape(4, 3).float() - a_patches = torch.ones((4, 3), dtype=torch.bool) - x_support = [torch.arange(6).reshape(2, 3)] - phi = torch.arange(15).reshape(5, 3).float() - sp, ss, na, nap, xs, nphi = nmf.reorder_sources( - order, source_params, source_seeds, a, a_patches, x_support, phi - ) - np.testing.assert_array_equal(ss[:, 0], [6.0, 0.0, 3.0]) - np.testing.assert_array_equal(na[0].numpy(), [2.0, 0.0, 1.0]) - np.testing.assert_array_equal(xs[0][0].numpy(), [2, 0, 1]) - - -class TestSolvePhiMotion(unittest.TestCase): - """Unregularized, rank-revealing QR temporal solve.""" - - def test_overdetermined_multiple_frames(self): - """QR matches an independent SVD least-squares reference.""" - for dtype in (torch.float32, torch.float64): - with self.subTest(dtype=dtype): - x = torch.tensor( - [[1, 2], [3, 1], [2, 4], [0, 1]], dtype=dtype - ) - data = torch.tensor( - [[2, 1], [0, 3], [4, 2], [1, 5]], dtype=dtype - ) - phi = nmf.solve_phi_motion(x, data) - expected = np.linalg.lstsq( - x.numpy(), data.numpy(), rcond=None - )[0].T - self.assertEqual(phi.shape, (2, 2)) - self.assertEqual(phi.dtype, dtype) - self.assertEqual(phi.device, x.device) - np.testing.assert_allclose( - phi.numpy(), expected, rtol=1e-5, atol=1e-6 - ) - - def test_small_profiles_are_not_regularized(self): - """Tiny profile magnitudes must not shrink the temporal weights.""" - x = 1e-6 * torch.tensor([[1., 0.], [0., 1.], [1., 1.]]) - expected = torch.tensor([[2., 3.], [4., 5.]]) - phi = nmf.solve_phi_motion(x, x @ expected.T) - torch.testing.assert_close(phi, expected) - - def test_nearly_collinear_profiles(self): - """QR resolves columns whose float32 normal equations lose rank.""" - x = torch.tensor([[1., 1.], [1., 1.0001], [1., 0.9999]]) - expected = torch.tensor([[2., -1.], [-3., 4.]]) - phi = nmf.solve_phi_motion(x, x @ expected.T) - torch.testing.assert_close(phi, expected, rtol=5e-3, atol=5e-3) - - def test_rank_deficient_basic_solution(self): - """Dependent coefficients are zero, not shared by minimum norm.""" - for dtype in (torch.float32, torch.float64): - with self.subTest(dtype=dtype): - x = torch.tensor([[1, 2], [2, 4], [3, 6]], dtype=dtype) - # Include an inconsistent RHS to test least squares too. - data = torch.tensor([[3, 1], [6, 0], [9, 0]], dtype=dtype) - phi = nmf.solve_phi_motion(x, data) - expected = torch.tensor( - [[0, 1.5], [0, 1 / 28]], dtype=dtype - ) - torch.testing.assert_close(phi, expected) - self.assertTrue(torch.all(phi[:, 0] == 0)) - # Residual is orthogonal to the column space. - torch.testing.assert_close( - x.T @ (x @ phi.T - data), - torch.zeros((2, 2), dtype=dtype), - atol=2e-5, rtol=0, - ) - - def test_underdetermined_basic_solution(self): - """Solve only the independent pivot columns and undo permutation.""" - for dtype in (torch.float32, torch.float64): - with self.subTest(dtype=dtype): - # Distinct pivot norms avoid ambiguous tie-breaking. - x = torch.tensor([[1, 0, 2], [0, 3, 0]], dtype=dtype) - data = torch.tensor([[4, -2], [9, 6]], dtype=dtype) - phi = nmf.solve_phi_motion(x, data) - expected = torch.tensor([[0, 3, 2], [0, 2, -1]], dtype=dtype) - torch.testing.assert_close(phi, expected) - self.assertTrue(torch.all(phi[:, 0] == 0)) - torch.testing.assert_close(x @ phi.T, data) - - def test_rank_deficient_underdetermined(self): - """A wide rank-one system uses exactly one pivot coefficient.""" - x = torch.tensor([[0., 1., 2.], [0., 2., 4.]]) - data = torch.tensor([[6., -2.], [12., -4.]]) - phi = nmf.solve_phi_motion(x, data) - torch.testing.assert_close( - phi, torch.tensor([[0., 0., 3.], [0., 0., -1.]]) - ) - self.assertTrue(torch.all(phi[:, :2] == 0)) - - def test_zero_rank(self): - """A zero design matrix returns the all-zero basic solution.""" - x = torch.zeros((3, 2)) - phi = nmf.solve_phi_motion(x, torch.ones((3, 4))) - torch.testing.assert_close(phi, torch.zeros((4, 2))) - - def test_nonfinite_frames_do_not_abort_finite_fits(self): - """Missing observations invalidate a frame, not the whole batch.""" - for dtype in (torch.float32, torch.float64): - for design in ( - [[1, 0], [0, 1], [1, 1]], - [[1, 2], [2, 4], [3, 6]], - [[1, 0, 2], [0, 3, 0]], - [[0, 0], [0, 0], [0, 0]], - ): - with self.subTest(dtype=dtype, design=design): - x = torch.tensor(design, dtype=dtype) - data = torch.arange( - x.shape[0] * 6, dtype=dtype - ).reshape(x.shape[0], 6) - data[0, 1] = float("nan") - data[:, 2] = float("nan") - data[-1, 3] = float("inf") - data[0, 4] = -float("inf") - original = data.clone() - expected = nmf.solve_phi_motion(x, data[:, [0, 5]]) - actual = nmf.solve_phi_motion(x, data) - self.assertEqual(actual.shape, (6, x.shape[1])) - self.assertEqual(actual.dtype, dtype) - torch.testing.assert_close(actual[[0, 5]], expected) - self.assertTrue(torch.isnan(actual[1:5]).all()) - torch.testing.assert_close(data, original, equal_nan=True) - - def test_all_frames_missing(self): - """An entirely missing bin returns NaNs, including dependent sources.""" - for dtype in (torch.float32, torch.float64): - with self.subTest(dtype=dtype): - x = torch.tensor([[1, 2], [2, 4], [3, 6]], dtype=dtype) - data = torch.full((3, 4), float("nan"), dtype=dtype) - actual = nmf.solve_phi_motion(x, data) - self.assertEqual(actual.shape, (4, 2)) - self.assertEqual(actual.dtype, dtype) - self.assertTrue(torch.isnan(actual).all()) - - def test_nonfinite_profiles_still_raise(self): - """Invalid spatial models must not be silently accepted by QR.""" - for value in (float("nan"), float("inf")): - with self.subTest(value=value): - x = torch.tensor([[1., 0.], [0., value], [1., 1.]]) - with self.assertRaisesRegex(ValueError, "infs or NaNs"): - nmf.solve_phi_motion(x, torch.ones((3, 2))) - - def test_numerically_dependent_column(self): - """The dtype-relative cutoff drops a tiny trailing QR diagonal.""" - for dtype in (torch.float32, torch.float64): - with self.subTest(dtype=dtype): - x = torch.tensor( - [[1, 0], [0, torch.finfo(dtype).eps / 10], [0, 0]], - dtype=dtype, - ) - data = x @ torch.ones((2, 1), dtype=dtype) - phi = nmf.solve_phi_motion(x, data) - torch.testing.assert_close( - phi, torch.tensor([[1, 0]], dtype=dtype) - ) - - def test_empty_sources_and_frames(self): - """Empty bins and fully pruned sources preserve output dimensions.""" - for n_sources, n_frames in ((0, 3), (2, 0), (0, 0)): - with self.subTest(n_sources=n_sources, n_frames=n_frames): - x = torch.eye(4, n_sources) - data = torch.zeros((4, n_frames)) - phi = nmf.solve_phi_motion(x, data) - self.assertEqual(phi.shape, (n_frames, n_sources)) - - -class TestPhiAndSnr(unittest.TestCase): - """fit_phi_all_motions / build_x_support_mots / compute_source_snr.""" - - def _setup(self, seed=0): - """Build A / data / geometry for a couple of sources.""" - g = _small_geometry(seed=seed) - npc, npr, nz = g["npc"], g["npr"], g["num_fast_zs"] - n_pix = nz * npc * npr - sel = g["sel_pix_idxs"] - pct = torch.tensor( - geometry.pixel_coords_from_idxs(sel, npc, npr), dtype=torch.float32 - ) - seeds = np.array([[0.0, 7.0, 7.0], [1.0, 7.0, 8.0]]) - params = nmf.init_source_params(seeds) - patches = nmf.build_a_patches(seeds, pct, sel, n_pix, d_xy=5) - a = nmf.project_spatial_profiles(params, pct, sel, n_pix, patches) - return g, a, sel, seeds - - def test_build_x_support_and_fit_phi(self): - """phi solves the per-motion least squares for structured data.""" - g, a, sel, seeds = self._setup() - n_motions = g["umyx"].shape[0] - supports = nmf.build_x_support_mots(g["h_mots"], a, sel, n_motions) - self.assertEqual(len(supports), n_motions) - - n_frames = 8 - mot_yx = np.array([0, 1] * (n_frames // 2), dtype=np.int32) - # structured data: exactly reconstructable from A with unit phi - data = torch.zeros((g["num_super_pixels"], n_frames)) - for i in range(n_motions): - frames = np.flatnonzero(mot_yx == i) - x = torch.sparse.mm(g["h_mots"][i], a[sel, :]) - data[:, frames] = x @ torch.ones((x.shape[1], len(frames))) - phi = torch.full((n_frames, seeds.shape[0]), float("nan")) - phi = nmf.fit_phi_all_motions( - a, g["h_mots"], sel, data, mot_yx, n_motions, phi - ) - # recovered phi should be ~1 for the fit frames - fit_frames = mot_yx >= 0 - np.testing.assert_allclose( - phi.numpy()[fit_frames], 1.0, rtol=1e-3, atol=1e-3 - ) - - def test_compute_source_snr_shape_and_high_for_signal(self): - """SNR is per-source and high when data is explained by the source.""" - g, a, sel, seeds = self._setup() - n_motions = g["umyx"].shape[0] - n_frames = 8 - mot_yx = np.array([0, 1] * (n_frames // 2), dtype=np.int32) - data = torch.zeros((g["num_super_pixels"], n_frames)) - phi = torch.full((n_frames, seeds.shape[0]), float("nan")) - for i in range(n_motions): - frames = np.flatnonzero(mot_yx == i) - x = torch.sparse.mm(g["h_mots"][i], a[sel, :]) - data[:, frames] = x @ torch.ones((x.shape[1], len(frames))) - phi = nmf.fit_phi_all_motions( - a, g["h_mots"], sel, data, mot_yx, n_motions, phi - ) - snr = nmf.compute_source_snr( - a, g["h_mots"], sel, data, phi, mot_yx, n_motions, seeds.shape[0] - ) - self.assertEqual(tuple(snr.shape), (2,)) - # data is fully explained -> residual ~0 -> very large SNR - self.assertTrue(torch.all(snr > 100)) - - -class TestLocalizeSources(unittest.TestCase): - """fit_sources end-to-end driver.""" - - def _inputs(self, seed=0): - """Small geometry + seeds + random residual (survives pruning).""" - g = _small_geometry(seed=seed) - npc, npr, nz = g["npc"], g["npr"], g["num_fast_zs"] - n_pix = nz * npc * npr - sel = g["sel_pix_idxs"] - coords = geometry.pixel_coords_from_idxs(sel, npc, npr) - seeds = np.array([[0.0, 7.0, 7.0], [0.0, 7.0, 9.0], [1.0, 7.0, 8.0]]) - rng = np.random.default_rng(seed + 100) - n_frames = 16 - residual = rng.standard_normal( - (g["num_super_pixels"], n_frames) - ).astype(np.float32) - mot_yx = np.array([0, 1] * (n_frames // 2), dtype=np.int32) - return g, seeds, residual, mot_yx, coords, n_pix - - def test_full_run_with_pruning(self): - """Three outer iterations exercise sparsify, sort, and prune.""" - g, seeds, residual, mot_yx, coords, n_pix = self._inputs() - torch.manual_seed(7) - out = nmf.fit_sources( - seeds.copy(), - residual, - g["h_mots"], - g["umyx"], - mot_yx, - g["sel_pix_idxs"], - coords, - n_pix, - d_xy=5, - sparse_fac=float(np.exp(-3.0)), - outer_loop_iters=3, - mult_nmf_max_iters=4, - ) - self.assertEqual(out["A"].shape[0], n_pix) - self.assertEqual(out["A"].shape[1], out["n_sources"]) - self.assertEqual(out["source_params"].shape[1], 6) - self.assertEqual(out["phi_low_res"].shape[0], residual.shape[1]) - self.assertEqual(out["source_snr"].shape[0], out["n_sources"]) - self.assertGreaterEqual(out["n_sources"], 1) - - def test_deterministic_under_seed(self): - """Same torch seed -> identical spatial profiles.""" - g, seeds, residual, mot_yx, coords, n_pix = self._inputs(seed=1) - kwargs = dict( - h_mots=g["h_mots"], - unique_motion_to_keep_yx=g["umyx"], - mot_inds_yx=mot_yx, - sel_pix_idxs=g["sel_pix_idxs"], - pixel_coords=coords, - n_pixels=n_pix, - d_xy=5, - sparse_fac=float(np.exp(-3.0)), - outer_loop_iters=2, - mult_nmf_max_iters=3, - ) - torch.manual_seed(3) - a1 = nmf.fit_sources(seeds.copy(), residual, **kwargs)["A"] - torch.manual_seed(3) - a2 = nmf.fit_sources(seeds.copy(), residual, **kwargs)["A"] - np.testing.assert_array_equal(a1.numpy(), a2.numpy()) - - def test_explicit_seed_ignores_global_rng(self): - """An explicit seed is reproducible whatever the global torch state.""" - g, seeds, residual, mot_yx, coords, n_pix = self._inputs(seed=1) - kwargs = dict( - h_mots=g["h_mots"], - unique_motion_to_keep_yx=g["umyx"], - mot_inds_yx=mot_yx, - sel_pix_idxs=g["sel_pix_idxs"], - pixel_coords=coords, - n_pixels=n_pix, - d_xy=5, - sparse_fac=float(np.exp(-3.0)), - outer_loop_iters=2, - mult_nmf_max_iters=3, - seed=11, - ) - torch.manual_seed(3) - first = nmf.fit_sources(seeds.copy(), residual, **kwargs) - global_state = torch.get_rng_state() - torch.manual_seed(99) - second = nmf.fit_sources(seeds.copy(), residual, **kwargs) - for key in ("A", "phi_low_res", "source_params", "source_snr"): - np.testing.assert_array_equal( - np.asarray(first[key]), np.asarray(second[key]), err_msg=key - ) - # The seeded path leaves the global RNG untouched. - torch.manual_seed(3) - nmf.fit_sources(seeds.copy(), residual, **kwargs) - self.assertTrue(torch.equal(torch.get_rng_state(), global_state)) - - def test_temporal_filter_matches_activity_filter(self): - """Reuse activity smoothing and omit its low-coverage edge frames.""" - g, seeds, residual, mot_yx, coords, n_pix = self._inputs(seed=1) - original = residual.copy() - kernel = activity.decay_kernel_1d(0.1, 10).astype(np.float32) - filtered = activity.smooth_rho(residual.copy(), kernel) - filtered_motions = mot_yx.copy() - filtered_motions[~np.all(np.isfinite(filtered), axis=0)] = -1 - self.assertTrue(np.any(filtered_motions < 0)) - self.assertFalse(np.allclose(filtered, residual)) - kwargs = dict( - h_mots=g["h_mots"], - unique_motion_to_keep_yx=g["umyx"], - sel_pix_idxs=g["sel_pix_idxs"], - pixel_coords=coords, - n_pixels=n_pix, - d_xy=5, - sparse_fac=float(np.exp(-3.0)), - outer_loop_iters=3, - mult_nmf_max_iters=3, - ) - torch.manual_seed(3) - expected = nmf.fit_sources( - seeds.copy(), filtered, mot_inds_yx=filtered_motions, **kwargs - ) - torch.manual_seed(3) - actual = nmf.fit_sources( - seeds.copy(), residual, mot_inds_yx=mot_yx, - temporal_kernel=kernel, **kwargs - ) - np.testing.assert_array_equal(residual, original) - self.assertEqual(actual["n_sources"], expected["n_sources"]) - for key in ("A", "phi_low_res", "source_params", "source_snr"): - np.testing.assert_allclose( - actual[key], expected[key], rtol=1e-5, atol=1e-6, - err_msg=key, - ) - np.testing.assert_array_equal( - actual["source_seeds"], expected["source_seeds"] - ) - - def test_temporal_filter_dropped_frames_survive_pruning(self): - """Missing samples must not poison otherwise usable source fits.""" - g, seeds, residual, mot_yx, coords, n_pix = self._inputs() - residual = np.tile(residual, (1, 16)) - mot_yx = np.tile(mot_yx, 16) - mot_yx[128] = -1 - original_motions = mot_yx.copy() - kernel = activity.decay_kernel_1d(0.15, 100) - for missing_value in (np.nan, 1e6): - with self.subTest(missing_value=missing_value): - residual[:, 128] = missing_value - original = residual.copy() - torch.manual_seed(7) - out = nmf.fit_sources( - seeds.copy(), residual, g["h_mots"], g["umyx"], - mot_yx, g["sel_pix_idxs"], coords, n_pix, - d_xy=5, sparse_fac=float(np.exp(-3.0)), - outer_loop_iters=3, mult_nmf_max_iters=4, - temporal_kernel=kernel, - ) - self.assertGreater(out["n_sources"], 0) - self.assertTrue(np.all(np.isfinite(out["source_snr"]))) - self.assertTrue(torch.all(torch.isfinite(out["A"]))) - self.assertTrue(torch.all(torch.isnan(out["phi_low_res"][128]))) - self.assertTrue(torch.any(torch.isfinite(out["phi_low_res"]))) - np.testing.assert_array_equal(residual, original) - np.testing.assert_array_equal(mot_yx, original_motions) - - def test_temporal_filter_excludes_empty_motion_bins(self): - """Low coverage can remove an entire bin without poisoning Adam.""" - g, seeds, residual, mot_yx, coords, n_pix = self._inputs() - # Only the first frame uses bin 0; a uniform kernel rejects both - # edges. An isolated row-wise NaN also rejects nearby frames. - mot_yx[:] = 1 - mot_yx[0] = 0 - residual[0, 8] = np.nan - kernel = np.ones(3) / 3 - filtered = activity.smooth_rho(residual.copy(), kernel) - expected_motions = np.full_like(mot_yx, -1) - expected_motions[np.all(np.isfinite(filtered), axis=0)] = 0 - kwargs = dict( - sel_pix_idxs=g["sel_pix_idxs"], pixel_coords=coords, - n_pixels=n_pix, d_xy=5, sparse_fac=0.05, - outer_loop_iters=1, mult_nmf_max_iters=3, - ) - torch.manual_seed(7) - expected = nmf.fit_sources( - seeds.copy(), filtered, [g["h_mots"][1]], g["umyx"][1:], - expected_motions, **kwargs, - ) - torch.manual_seed(7) - actual = nmf.fit_sources( - seeds.copy(), residual, g["h_mots"], g["umyx"], mot_yx, - temporal_kernel=kernel, **kwargs, - ) - self.assertTrue(torch.all(torch.isfinite(actual["A"]))) - for key in ("A", "phi_low_res", "source_params", "source_snr"): - np.testing.assert_allclose(actual[key], expected[key]) - excluded = expected_motions < 0 - self.assertTrue( - torch.all(torch.isnan(actual["phi_low_res"][excluded])) - ) - - def test_temporal_filter_no_usable_frames(self): - """Report missing data explicitly instead of silently pruning all seeds.""" - g, seeds, residual, mot_yx, coords, n_pix = self._inputs() - residual[:] = np.nan - with self.assertRaisesRegex(ValueError, "No usable frames remain"): - nmf.fit_sources( - seeds, residual, g["h_mots"], g["umyx"], mot_yx, - g["sel_pix_idxs"], coords, n_pix, d_xy=5, - sparse_fac=0.05, temporal_kernel=np.ones(1), - ) - - def test_adam_early_convergence_break(self): - """A huge gd_tol triggers the Adam convergence break.""" - g, seeds, residual, mot_yx, coords, n_pix = self._inputs(seed=2) - torch.manual_seed(5) - out = nmf.fit_sources( - seeds.copy(), - residual, - g["h_mots"], - g["umyx"], - mot_yx, - g["sel_pix_idxs"], - coords, - n_pix, - d_xy=5, - sparse_fac=float(np.exp(-3.0)), - outer_loop_iters=1, - mult_nmf_max_iters=2, - gd_tol=1e9, - ) - self.assertEqual(out["n_sources"], 3) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_bandsilo_peaks.py b/tests/test_bandsilo_peaks.py deleted file mode 100644 index b45f8d7..0000000 --- a/tests/test_bandsilo_peaks.py +++ /dev/null @@ -1,726 +0,0 @@ -"""Tests for giant_python.numerics.peaks (Phase 5 peak detection). - -Covers the integrated-Gaussian forward/Jacobian kernels, the bounded LM -solver, the per-plane detector, and the 3-D ``get_act_im_peaks`` driver with -synthetic activity images. The full detector pipeline is additionally -cross-checked against a verbatim copy of the reference in the project's -development notes; here we assert structural/numerical behavior on small cases. -""" - -import unittest -from unittest.mock import patch - -import numpy as np - -from giant_python.numerics import peaks as pk - - -def _grid(height, width): - """Return an ``(H*W, 2)`` array of all ``[y, x]`` pixel centers.""" - yy, xx = np.mgrid[0:height, 0:width] - return np.column_stack( - [yy.ravel().astype(float), xx.ravel().astype(float)] - ) - - -def _single_gaussian_plane(height, width, cy, cx, amp, sigma, sigma_x=0.8): - """Render one integrated Gaussian over an ``(H, W)`` plane.""" - yx = _grid(height, width) - theta = np.array([[amp, cy, cx, sigma, sigma_x]]) - return pk.gaussian_peaks_integrated(theta, yx).reshape(height, width) - - -class TestGaussianForward(unittest.TestCase): - """gaussian_peaks_integrated analytic properties.""" - - def test_anisotropic_total_integral(self): - """Unequal widths integrate to amp*2*pi*sigma_y*sigma_x.""" - theta = np.array([[3.0, 50.3, 12.7, 8.0, 0.7]]) - total = pk.gaussian_peaks_integrated(theta, _grid(101, 27)).sum() - self.assertAlmostEqual(total, 3.0 * 2 * np.pi * 8.0 * 0.7, places=6) - - def test_four_column_parameters_rejected(self): - """A missing x width fails instead of silently sharing the y width.""" - yx = _grid(25, 25) - theta = np.array([[2.0, 12.3, 10.7, 0.9]]) - with self.assertRaisesRegex(ValueError, r"shape \(N, 5\).*sigma_x"): - pk.gaussian_peaks_integrated(theta, yx) - - def test_total_integral(self): - """Integrated Gaussian summed over a wide grid gives amp*2*pi*s^2.""" - yx = _grid(60, 60) - amp, s = 3.0, 2.0 - theta = np.array([[amp, 30.0, 30.0, s, s]]) - total = pk.gaussian_peaks_integrated(theta, yx).sum() - self.assertAlmostEqual(total, amp * 2 * np.pi * s**2, places=3) - - def test_superposition(self): - """Two Gaussians sum to the sum of their individual renders.""" - yx = _grid(30, 30) - t1 = np.array([[2.0, 10.0, 12.0, 1.5, 1.5]]) - t2 = np.array([[3.0, 20.0, 18.0, 1.0, 1.0]]) - both = pk.gaussian_peaks_integrated(np.vstack([t1, t2]), yx) - sep = pk.gaussian_peaks_integrated( - t1, yx - ) + pk.gaussian_peaks_integrated(t2, yx) - np.testing.assert_allclose(both, sep, rtol=1e-10) - - -class TestParameterContract(unittest.TestCase): - """Gaussian kernels and both fit stages require five explicit columns.""" - - def test_invalid_parameter_shapes_rejected(self): - """Reject missing widths, extra columns and non-matrix inputs early.""" - yx = _grid(3, 3) - for shape in [(1, 4), (0, 4), (1, 6), (1, 0), (5,), (1, 1, 5)]: - theta = np.ones(shape) - for kernel in [ - pk.gaussian_peaks_integrated, - pk._gaussian_peaks_integrated_val_jac, - pk._lsq_curvefit, - pk._fit_isotropic_then_anisotropic, - ]: - with self.subTest(shape=shape, kernel=kernel.__name__): - args = (theta, yx) - if kernel in [ - pk._lsq_curvefit, - pk._fit_isotropic_then_anisotropic, - ]: - args += ( - np.zeros(9), - np.zeros(theta.size), - np.ones(theta.size), - ) - with self.assertRaisesRegex( - ValueError, r"shape \(N, 5\).*sigma_y, sigma_x" - ): - kernel(*args) - - def test_bounds_cannot_broadcast(self): - """Fitters reject scalar, four-column and unflattened bounds.""" - theta = np.array([[1.0, 1.0, 1.0, 0.5, 0.5]]) - for fitter in [pk._lsq_curvefit, pk._fit_isotropic_then_anisotropic]: - for shape in [(), (1,), (4,), (1, 5)]: - for invalid_lower in [True, False]: - lb = np.zeros(shape if invalid_lower else (5,)) - ub = np.ones((5,) if invalid_lower else shape) - with self.subTest( - fitter=fitter.__name__, - shape=shape, - invalid_lower=invalid_lower, - ): - with self.assertRaisesRegex( - ValueError, r"bounds must have shape \(5\*N,\)" - ): - fitter(theta, _grid(3, 3), np.zeros(9), lb, ub) - - -class TestValJac(unittest.TestCase): - """_gaussian_peaks_integrated_val_jac value + analytical Jacobian.""" - - def test_anisotropic_jacobian_matches_finite_difference(self): - """Both width derivatives are independent and correctly ordered.""" - yx = _grid(35, 19) - theta = np.array( - [ - [2.0, 16.3, 7.7, 8.0, 0.7], - [1.2, 21.0, 12.0, 3.0, 1.0], - ] - ) - val, jac = pk._gaussian_peaks_integrated_val_jac(theta, yx) - np.testing.assert_allclose( - val, pk.gaussian_peaks_integrated(theta, yx), rtol=1e-12 - ) - self.assertEqual(jac.shape, (len(yx), theta.size)) - eps = 1e-6 - for k in range(theta.size): - hi = theta.ravel().copy() - lo = theta.ravel().copy() - hi[k] += eps - lo[k] -= eps - fd = ( - pk.gaussian_peaks_integrated(hi.reshape(-1, 5), yx) - - pk.gaussian_peaks_integrated(lo.reshape(-1, 5), yx) - ) / (2 * eps) - np.testing.assert_allclose(jac[:, k], fd, rtol=1e-4, atol=1e-7) - - def test_value_matches_forward(self): - """The val_jac value equals the forward-only evaluation.""" - yx = _grid(20, 20) - theta = np.array( - [[2.0, 9.3, 11.7, 1.4, 1.4], [1.5, 14.0, 6.0, 1.1, 1.1]] - ) - val, _ = pk._gaussian_peaks_integrated_val_jac(theta, yx) - fwd = pk.gaussian_peaks_integrated(theta, yx) - np.testing.assert_allclose(val, fwd, rtol=1e-12) - - def test_jacobian_matches_finite_difference(self): - """Analytical Jacobian matches a central finite-difference estimate.""" - yx = _grid(16, 16) - theta = np.array( - [[2.0, 8.3, 7.7, 1.3, 1.3], [1.2, 11.0, 5.0, 0.9, 0.9]] - ) - _, jac = pk._gaussian_peaks_integrated_val_jac(theta, yx) - flat = theta.ravel().copy() - eps = 1e-6 - for k in range(flat.size): - hi = flat.copy() - hi[k] += eps - lo = flat.copy() - lo[k] -= eps - v_hi = pk.gaussian_peaks_integrated(hi.reshape(-1, 5), yx) - v_lo = pk.gaussian_peaks_integrated(lo.reshape(-1, 5), yx) - fd = (v_hi - v_lo) / (2 * eps) - np.testing.assert_allclose(jac[:, k], fd, rtol=1e-4, atol=1e-4) - - -class TestLsqCurvefit(unittest.TestCase): - """_lsq_curvefit recovery and control-flow branches.""" - - def test_recovers_known_gaussian(self): - """Fitting noise-free data recovers the generating parameters.""" - yx = _grid(21, 21) - true = np.array([[5.0, 10.4, 9.6, 6.0, 0.7]]) - ydata = pk.gaussian_peaks_integrated(true, yx) - theta0 = np.array([[3.0, 10.0, 10.0, 0.5, 0.5]]) - lb, ub = pk._make_bounds(theta0[:, 1:3], 21, 21) - fit = pk._lsq_curvefit(theta0, yx, ydata, lb, ub) - np.testing.assert_allclose(fit, true, atol=1e-2) - - def test_width_limits(self): - """A target wider than either limit cannot exceed the fitted bounds.""" - yx = _grid(41, 21) - true = np.array([[5.0, 20.4, 9.6, 10.0, 1.5]]) - theta0 = np.array([[4.0, 20.0, 10.0, 6.0, 0.8]]) - lb, ub = pk._make_bounds(theta0[:, 1:3], 41, 21) - fit = pk._lsq_curvefit( - theta0, yx, pk.gaussian_peaks_integrated(true, yx), lb, ub - ) - self.assertTrue(np.all(fit.ravel() >= lb)) - self.assertTrue(np.all(fit.ravel() <= ub)) - np.testing.assert_allclose(fit[0, 3:], [8.0, 1.0], atol=1e-2) - - def test_max_nfev_early_break(self): - """max_nfev=1 returns the (clipped) initial parameters immediately.""" - yx = _grid(11, 11) - ydata = pk.gaussian_peaks_integrated( - np.array([[4.0, 5.0, 5.0, 1.0, 1.0]]), yx - ) - theta0 = np.array([[3.0, 5.0, 5.0, 1.0, 1.0]]) - lb, ub = pk._make_bounds(theta0[:, 1:3], 11, 11) - out = pk._lsq_curvefit(theta0, yx, ydata, lb, ub, max_nfev=1) - np.testing.assert_allclose(out, theta0) - - def test_non_improving_step_increases_damping(self): - """A clipped, non-improving step exercises the damping-increase branch. - - Tight mean bounds around a wrong location force steps that cannot - reduce cost, so the solver repeatedly raises the LM damping. - """ - yx = _grid(21, 21) - # data has its peak far from the seed; bounds pin the mean near seed - ydata = pk.gaussian_peaks_integrated( - np.array([[6.0, 15.0, 15.0, 1.2, 1.2]]), yx - ) - theta0 = np.array([[6.0, 4.0, 4.0, 1.2, 1.2]]) - lb = np.array([5.9, 3.9, 3.9, 1.1, 1.1]) - ub = np.array([6.1, 4.1, 4.1, 1.3, 1.3]) - out = pk._lsq_curvefit(theta0, yx, ydata, lb, ub, max_nfev=50) - # stays within the tight bounds (no crash, no bound violation) - self.assertTrue(np.all(out.ravel() >= lb - 1e-9)) - self.assertTrue(np.all(out.ravel() <= ub + 1e-9)) - - -class TestIsotropicWarmStart(unittest.TestCase): - """Two-stage fitting preserves round peaks while permitting elongation.""" - - def test_shared_width_fit(self): - """The constrained LM stage recovers a round peak in five columns.""" - yx = _grid(21, 21) - true = np.array([[5.0, 10.3, 9.7, 0.8, 0.8]]) - theta0 = np.array([[3.0, 10.0, 10.0, 0.5, 0.5]]) - lb, ub = pk._make_bounds(theta0[:, 1:3], 21, 21) - ub[3] = ub[4] - out = pk._lsq_curvefit( - theta0, - yx, - pk.gaussian_peaks_integrated(true, yx), - lb, - ub, - shared_widths=True, - ) - np.testing.assert_allclose(out, true, atol=0.01) - np.testing.assert_array_equal(out[:, 3], out[:, 4]) - - def test_shared_width_constraint_requires_matching_columns(self): - """Tied updates must not silently reinterpret unequal inputs/bounds.""" - for mismatch in ["theta", "lower", "upper"]: - theta = np.array([[1.0, 1.0, 1.0, 0.5, 0.5]]) - lb = np.array([0.0, 0.0, 0.0, 0.35, 0.35]) - ub = np.array([2.0, 2.0, 2.0, 1.0, 1.0]) - if mismatch == "theta": - theta[0, 4] = 0.6 - elif mismatch == "lower": - lb[4] = 0.4 - else: - ub[4] = 0.9 - with self.subTest(mismatch=mismatch): - with self.assertRaisesRegex( - ValueError, "requires equal y/x widths and bounds" - ): - pk._lsq_curvefit( - theta, - _grid(3, 3), - np.zeros(9), - lb, - ub, - shared_widths=True, - ) - - def test_stages_and_bounds(self): - """Relaxation starts from the fitted shared widths, not the seed.""" - theta0 = np.array([[3.0, 10.0, 10.0, 4.0, 0.8]]) - original = theta0.copy() - lb, ub = pk._make_bounds(theta0[:, 1:3], 21, 21) - iso_fit = np.array([[5.0, 10.3, 9.7, 0.9, 0.9]]) - final_fit = np.array([[5.0, 10.3, 9.7, 4.0, 0.7]]) - with patch.object( - pk, "_lsq_curvefit", side_effect=[iso_fit, final_fit] - ) as solver: - out = pk._fit_isotropic_then_anisotropic( - theta0, - _grid(21, 21), - np.zeros(441), - lb, - ub, - max_nfev=100, - ) - self.assertEqual(solver.call_count, 2) - first, second = solver.call_args_list - self.assertEqual(first.args[0].shape, (1, 5)) - np.testing.assert_array_equal(first.args[0][:, 3], first.args[0][:, 4]) - np.testing.assert_array_equal(first.args[3][3:], [0.35, 0.35]) - np.testing.assert_array_equal(first.args[4][3:], [1.0, 1.0]) - self.assertTrue(first.kwargs["shared_widths"]) - self.assertNotIn("shared_widths", second.kwargs) - np.testing.assert_array_equal( - second.args[0], [[5.0, 10.3, 9.7, 0.9, 0.9]] - ) - np.testing.assert_array_equal(second.args[3], lb) - np.testing.assert_array_equal(second.args[4], ub) - self.assertEqual(first.kwargs["max_nfev"], 100) - self.assertEqual(second.kwargs["max_nfev"], 100) - np.testing.assert_array_equal(out, final_fit) - np.testing.assert_array_equal(theta0, original) - - def test_recovers_round_and_elongated_peaks(self): - """Joint fitting recovers round and tall peaks after relaxation.""" - yx = _grid(65, 31) - true = np.array( - [ - [5.0, 15.3, 8.7, 0.8, 0.8], - [8.0, 40.4, 22.2, 7.0, 0.7], - ] - ) - theta0 = np.array( - [ - [3.0, 15.0, 9.0, 0.5, 0.5], - [4.0, 40.0, 22.0, 0.5, 0.5], - ] - ) - lb, ub = pk._make_bounds(theta0[:, 1:3], 65, 31) - out = pk._fit_isotropic_then_anisotropic( - theta0, yx, pk.gaussian_peaks_integrated(true, yx), lb, ub - ) - np.testing.assert_allclose(out, true, atol=0.01) - - def test_exact_isotropic_minimum_stays_isotropic(self): - """Relaxing an already exact round fit does not change it.""" - theta = np.array([[5.0, 10.3, 9.7, 0.8, 0.8]]) - yx = _grid(21, 21) - lb, ub = pk._make_bounds(theta[:, 1:3], 21, 21) - out = pk._fit_isotropic_then_anisotropic( - theta, yx, pk.gaussian_peaks_integrated(theta, yx), lb, ub - ) - np.testing.assert_array_equal(out, theta) - - def test_empty_parameters(self): - """No peaks need no optimization.""" - theta = np.empty((0, 5)) - out = pk._fit_isotropic_then_anisotropic( - theta, _grid(3, 3), np.zeros(9), np.array([]), np.array([]) - ) - self.assertEqual(out.shape, (0, 5)) - - -class TestSmallHelpers(unittest.TestCase): - """_make_bounds / _peak_mask / _buffer_mask.""" - - def test_make_bounds(self): - """Means clip to the image; sigma_y <= 8 and sigma_x <= 1.""" - plocs = np.array([[0.0, 9.0]]) - lb, ub = pk._make_bounds(plocs, 10, 10) - # y lower clipped to 0; x within range - self.assertAlmostEqual(lb[1], 0.0) - self.assertAlmostEqual(ub[2], 9.0) # min(W-1, 9+1.5) = 9 - np.testing.assert_array_equal(lb[3:], [0.35, 0.35]) - np.testing.assert_array_equal(ub[3:], [8.0, 1.0]) - - def test_peak_mask_and_empty(self): - """Peak centers mark the mask; an empty parameter set marks nothing.""" - tf = np.array([[1.0, 3.2, 4.8, 1.0, 1.0]]) - mask = pk._peak_mask(tf, 8, 8) - self.assertTrue(mask[3, 5]) - self.assertEqual(mask.sum(), 1) - empty = pk._peak_mask(np.zeros((0, 5)), 8, 8) - self.assertFalse(empty.any()) - - def test_buffer_mask(self): - """buffer_size>0 dilates; <=0 returns the mask unchanged.""" - pim = np.zeros((7, 7), dtype=bool) - pim[3, 3] = True - buffered = pk._buffer_mask(pim, 3) - self.assertEqual(buffered.sum(), 9) # 3x3 dilation - same = pk._buffer_mask(pim, 0) - np.testing.assert_array_equal(same, pim) - - -class TestProcessCcNewPeak(unittest.TestCase): - """_process_cc_new_peak accept / reject (with and without other peaks).""" - - def test_rejects_integer_peak_with_others(self): - """A peak that converges to its integer seed, beside an existing one, - is rejected (and the others refit).""" - h = w = 15 - # a blob centered exactly on the integer seed -> fit stays near-integer - act = _single_gaussian_plane(h, w, 5.0, 5.0, 8.0, 1.1) - labeled = np.ones((h, w), dtype=int) - thetaf = np.array([[1.0, 10.0, 10.0, 1.0, 0.8]]) # existing peak in cc - p_locs = np.array([[10.0, 10.0]]) - reject_mask = np.zeros((h, w), dtype=bool) - new_thetaf, _, _, _ = pk._process_cc_new_peak( - ((5, 5), 1), thetaf, p_locs, labeled, act, 0.0, reject_mask, h, w - ) - self.assertEqual(new_thetaf.shape[0], 1) # new peak removed - self.assertTrue(reject_mask[5, 5]) - - def test_rejects_flat_peak_alone(self): - """A non-moving lone peak is rejected (no other peaks to refit).""" - h = w = 15 - act = np.zeros((h, w)) - labeled = np.zeros((h, w), dtype=int) - labeled[3:12, 3:12] = 1 - thetaf = np.zeros((0, 5)) - p_locs = np.zeros((0, 2)) - reject_mask = np.zeros((h, w), dtype=bool) - new_thetaf, _, _, _ = pk._process_cc_new_peak( - ((5, 5), 1), thetaf, p_locs, labeled, act, 0.0, reject_mask, h, w - ) - self.assertEqual(new_thetaf.shape[0], 0) - self.assertTrue(reject_mask[5, 5]) - - def test_accepts_moving_peak(self): - """A peak that moves off its integer seed is kept.""" - h = w = 15 - act = _single_gaussian_plane(h, w, 5.4, 5.6, 6.0, 1.1) - labeled = np.ones((h, w), dtype=int) - thetaf = np.zeros((0, 5)) - p_locs = np.zeros((0, 2)) - reject_mask = np.zeros((h, w), dtype=bool) - new_thetaf, _, _, _ = pk._process_cc_new_peak( - ((5, 5), 1), thetaf, p_locs, labeled, act, 0.0, reject_mask, h, w - ) - self.assertEqual(new_thetaf.shape[0], 1) - self.assertFalse(reject_mask.any()) - # the kept peak moved toward the true (off-integer) center - self.assertGreater( - abs(new_thetaf[0, 1] - round(new_thetaf[0, 1])), 1e-3 - ) - - -class TestRefineResidualPeaks(unittest.TestCase): - """_refine_residual_peaks early-exit branch.""" - - def test_empty_support_returns_unchanged(self): - """No support (labels max 0) exits immediately, returning thetaf.""" - h = w = 10 - thetaf = np.array([[1.0, 5.0, 5.0, 1.0, 0.8]]) - p_locs = np.array([[5.0, 5.0]]) - act = np.zeros((h, w)) - empty2d = np.zeros((h, w), dtype=bool) - out = pk._refine_residual_peaks( - thetaf, - p_locs, - act, - empty2d, # exclusion - 0.0, - 1.0, - 3.0, - 0, - h, - w, - empty2d.copy(), # buffer_mask - np.zeros((h, w)), # res_im - np.zeros((h, w), dtype=bool), # fit_support - np.zeros((h, w), dtype=bool), # act_sel_pix -> no labels - np.zeros((h, w)), # fit_im - ) - np.testing.assert_array_equal(out, thetaf) - - -class TestDetectPeaks2d(unittest.TestCase): - """detect_peaks_2d single-plane behavior.""" - - def test_no_peaks_returns_empty(self): - """A plane with nothing above threshold returns no peaks.""" - rng = np.random.default_rng(0) - act = rng.standard_normal((30, 30)) * 0.2 - out = pk.detect_peaks_2d( - act, - np.zeros((30, 30), dtype=bool), - mu_bg=0.0, - sigma_bg=0.2, - peak_thresh=100.0, # unreachable - peak_th=100.0, - ) - self.assertEqual(out.shape, (0, 5)) - - def test_elongated_blob_detected(self): - """A tall, narrow blob is fit as one anisotropic Gaussian.""" - act = _single_gaussian_plane(65, 25, 32.3, 12.4, 10.0, 7.0, 0.7) - out = pk.detect_peaks_2d( - act, - np.zeros_like(act, dtype=bool), - mu_bg=0.0, - sigma_bg=0.05, - peak_thresh=1.0, - peak_th=3.0, - ) - self.assertEqual(out.shape, (1, 5)) - np.testing.assert_allclose( - out[0], [10.0, 32.3, 12.4, 7.0, 0.7], atol=0.05 - ) - - def test_single_blob_detected(self): - """A single blob yields one peak near its true center.""" - act = _single_gaussian_plane(30, 30, 14.3, 16.7, 10.0, 1.3) - out = pk.detect_peaks_2d( - act, - np.zeros((30, 30), dtype=bool), - mu_bg=0.0, - sigma_bg=0.05, - peak_thresh=1.0, - peak_th=3.0, - ) - self.assertGreaterEqual(out.shape[0], 1) - # the strongest peak should sit near the true center - best = out[np.argmax(out[:, 0])] - self.assertAlmostEqual(best[1], 14.3, delta=0.5) - self.assertAlmostEqual(best[2], 16.7, delta=0.5) - - def test_close_pair_adds_residual_peak(self): - """Two blobs too close to both be initial maxima: the second is - recovered by the residual-peak refinement loop.""" - act = _single_gaussian_plane( - 34, 34, 16.4, 15.0, 10.0, 1.2 - ) + _single_gaussian_plane(34, 34, 16.4, 16.8, 9.0, 1.2) - out = pk.detect_peaks_2d( - act, - np.zeros((34, 34), dtype=bool), - mu_bg=0.0, - sigma_bg=0.05, - peak_thresh=1.0, - peak_th=3.0, - ) - # both blobs recovered though only one initial local max exists - self.assertEqual(out.shape[0], 2) - - -class TestGetActImPeaks(unittest.TestCase): - """get_act_im_peaks 3-D driver and its guard branches.""" - - def test_component_local_statistics_and_coordinates(self): - """Each component and plane gets its own median, MAD and threshold.""" - act = np.full((2, 16, 29), np.nan) - base = np.linspace(-1.0, 1.0, 100).reshape(10, 10) - regions = [ - (0, 2, 3, base), - (0, 4, 18, 5.0 + 8.0 * base), - (1, 2, 3, 2.0 + 3.0 * base), - ] - for z, y, x, values in regions: - act[z, y : y + 10, x : x + 10] = values - original = act.copy() - fitted = np.array([[10.0, 4.2, 5.3, 0.5, 0.5]]) - with patch.object( - pk, "detect_peaks_2d", return_value=fitted - ) as detect: - seeds = pk.get_act_im_peaks(act, peak_th=4.0, buffer_size=3) - - self.assertEqual(detect.call_count, 3) - expected_seeds = [] - for call, (z, y, x, values) in zip(detect.call_args_list, regions): - image, mask, mu, sigma, threshold, peak_th = call.args - expected_mu = np.median(values) - expected_sigma = np.median(np.abs(values - expected_mu)) / 0.67449 - np.testing.assert_array_equal(image, values) - self.assertFalse(mask.any()) - self.assertAlmostEqual(mu, expected_mu) - self.assertAlmostEqual(sigma, expected_sigma) - self.assertAlmostEqual(threshold, expected_mu + 4 * expected_sigma) - self.assertEqual(peak_th, 4.0) - self.assertEqual(call.kwargs, {"buffer_size": 3}) - expected_seeds.append([z, y + 4.2, x + 5.3]) - np.testing.assert_allclose(seeds, expected_seeds) - np.testing.assert_array_equal(act, original) - - def test_minimum_size_is_per_plane_and_inclusive(self): - """99-pixel regions are skipped even across z; 100 pixels qualifies.""" - act = np.full((2, 12, 23), np.nan) - act[:, :9, :11] = np.arange(99).reshape(9, 11) - act[0, :10, 13:23] = np.arange(100).reshape(10, 10) - with patch.object( - pk, "detect_peaks_2d", return_value=np.zeros((0, 5)) - ) as detect: - seeds = pk.get_act_im_peaks(act) - self.assertEqual(detect.call_count, 1) - self.assertEqual(detect.call_args.args[0].shape, (10, 10)) - self.assertEqual(seeds.shape, (0, 3)) - - def test_diagonal_regions_are_not_connected(self): - """Diagonal contact does not merge two sub-100-pixel regions.""" - act = np.full((1, 18, 18), np.nan) - act[0, :9, :9] = np.arange(81).reshape(9, 9) - act[0, 9:, 9:] = np.arange(81).reshape(9, 9) - with patch.object(pk, "detect_peaks_2d") as detect: - seeds = pk.get_act_im_peaks(act) - detect.assert_not_called() - self.assertEqual(seeds.shape, (0, 3)) - - def test_nested_components_and_exclusion_masks(self): - """Regions never share fit pixels; exclusions do not split CCs.""" - act = np.arange(900, dtype=float).reshape(1, 30, 30) - act[0, 8:22, 8:22] = np.inf - act[0, 10:20, 10:20] = np.arange(100).reshape(10, 10) - outer = np.isfinite(act[0]) - outer[10:20, 10:20] = False - for shape in [(30, 30), (1, 30, 30)]: - with self.subTest(mask_shape=shape): - exclusion = np.zeros(shape, dtype=bool) - exclusion[..., 14:16, :] = True - mask = exclusion if exclusion.ndim == 2 else exclusion[0] - with patch.object( - pk, "detect_peaks_2d", return_value=np.zeros((0, 5)) - ) as detect: - pk.get_act_im_peaks(act, exclusion_mask=exclusion) - self.assertEqual(detect.call_count, 2) - outer_call, inner_call = detect.call_args_list - np.testing.assert_array_equal( - np.isfinite(outer_call.args[0]), outer - ) - np.testing.assert_array_equal(outer_call.args[1], mask) - self.assertAlmostEqual( - outer_call.args[2], np.median(act[0][outer]) - ) - np.testing.assert_array_equal( - inner_call.args[0], act[0, 10:20, 10:20] - ) - np.testing.assert_array_equal( - inner_call.args[1], mask[10:20, 10:20] - ) - - def test_zero_mad_component_does_not_skip_other_components(self): - """A constant region is skipped independently of a noisy region.""" - act = np.ones((1, 10, 21)) - act[0, :, 10] = np.nan - act[0, :, 11:] = np.arange(100).reshape(10, 10) - with patch.object( - pk, "detect_peaks_2d", return_value=np.zeros((0, 5)) - ) as detect: - pk.get_act_im_peaks(act) - self.assertEqual(detect.call_count, 1) - self.assertAlmostEqual(detect.call_args.args[2], 49.5) - - def test_quiet_component_peak_survives_noisy_neighbor(self): - """A large noisy region cannot hide a quiet region's genuine peak.""" - rng = np.random.default_rng(17) - act = np.full((1, 80, 112), np.nan) - act[0, 5:35, 4:34] = rng.normal(0, 0.03, (30, 30)) - act[0, 5:35, 4:34] += _single_gaussian_plane( - 30, 30, 14.3, 16.7, 3.0, 0.8 - ) - act[0, :, 36:] = rng.normal(0, 2.0, (80, 76)) - seeds = pk.get_act_im_peaks(act, peak_th=6.0, buffer_size=3) - self.assertEqual(seeds.shape, (1, 3)) - np.testing.assert_allclose(seeds[0], [0, 19.3, 20.7], atol=0.15) - - def test_elongated_blob_seed(self): - """The 3-D driver retains its [z, y, x] output for unequal widths.""" - rng = np.random.default_rng(42) - act = rng.normal(0.0, 0.01, (1, 65, 25)) - act[0] += _single_gaussian_plane(65, 25, 32.3, 12.4, 10.0, 7.0, 0.7) - seeds = pk.get_act_im_peaks(act, peak_th=6.0, buffer_size=3) - self.assertEqual(seeds.shape, (1, 3)) - np.testing.assert_allclose(seeds[0], [0.0, 32.3, 12.4], atol=0.1) - - def _act_im(self, seed=0): - """Two-plane activity image with a blob per plane, low noise, NaNs.""" - rng = np.random.default_rng(seed) - act = rng.standard_normal((2, 40, 40)) * 0.3 - act[0] += _single_gaussian_plane(40, 40, 12.4, 18.6, 9.0, 1.3) - act[1] += _single_gaussian_plane(40, 40, 25.0, 15.0, 8.0, 1.2) - act[:, :3, :3] = np.nan - return act - - def test_detects_seeds_per_plane(self): - """Both planes' blobs are found, tagged with their z index.""" - seeds = pk.get_act_im_peaks(self._act_im(), peak_th=4.0, buffer_size=3) - self.assertEqual(seeds.shape[1], 3) - self.assertTrue(np.any(seeds[:, 0] == 0)) - self.assertTrue(np.any(seeds[:, 0] == 1)) - - def test_exclusion_mask_2d(self): - """A 2-D exclusion mask suppresses seeds in the masked region.""" - excl = np.zeros((40, 40), dtype=bool) - excl[8:18, 14:24] = True # covers the plane-0 blob - seeds = pk.get_act_im_peaks( - self._act_im(), peak_th=4.0, exclusion_mask=excl, buffer_size=3 - ) - # no seed inside the excluded box - for z, y, x in seeds: - self.assertFalse(8 <= y < 18 and 14 <= x < 24) - - def test_exclusion_mask_3d(self): - """A 3-D per-plane exclusion mask is applied plane-by-plane.""" - excl = np.zeros((2, 40, 40), dtype=bool) - excl[1, 20:30, 10:20] = True # covers the plane-1 blob only - seeds = pk.get_act_im_peaks( - self._act_im(), peak_th=4.0, exclusion_mask=excl, buffer_size=3 - ) - z1_in_box = any( - int(z) == 1 and 20 <= y < 30 and 10 <= x < 20 for z, y, x in seeds - ) - self.assertFalse(z1_in_box) - - def test_all_nan_returns_empty(self): - """An all-NaN activity image yields no seeds.""" - act = np.full((2, 20, 20), np.nan) - out = pk.get_act_im_peaks(act) - self.assertEqual(out.shape, (0, 3)) - - def test_zero_mad_returns_empty(self): - """A constant image (MAD sigma 0) yields no seeds.""" - act = np.ones((2, 20, 20)) - out = pk.get_act_im_peaks(act) - self.assertEqual(out.shape, (0, 3)) - - def test_no_peaks_returns_empty(self): - """Pure low noise with a high threshold yields no seeds.""" - rng = np.random.default_rng(1) - act = rng.standard_normal((2, 30, 30)) * 0.3 - out = pk.get_act_im_peaks(act, peak_th=50.0) - self.assertEqual(out.shape, (0, 3)) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_bandsilo_phase_filter.py b/tests/test_bandsilo_phase_filter.py deleted file mode 100644 index 9d2972c..0000000 --- a/tests/test_bandsilo_phase_filter.py +++ /dev/null @@ -1,361 +0,0 @@ -"""Phase-matched background medians, including overlapping band columns.""" - -import unittest - -import numpy as np - -from giant_python.extraction.band import summary_images as si - - -# Preserve the independent scientific reference's explicit sampling loops. -def brute_median( # noqa: C901 - image, - selected, - ref_r, - ref_c, - ref_d, - height, - width, - tolerance, - minimum, -): - """Compute a small-grid reference with explicit distance/window loops.""" - phases = np.full(image.shape, np.nan) - med = np.full(image.shape, np.nan) - for flat in selected: - z, r, c = np.unravel_index(flat, image.shape) - centers = np.unique(ref_r[(ref_d == z) & (ref_c == c)]) - if not centers.size: - continue - distances = np.abs(r - centers) - phases[z, r, c] = r - centers[np.argmin(distances)] - for flat in selected: - z, r, c = np.unravel_index(flat, image.shape) - if not np.isfinite(phases[z, r, c]) or not np.isfinite(image[z, r, c]): - continue - samples = [] - for sr in range( - max(0, r - height // 2), min(image.shape[1], r + height // 2 + 1) - ): - for sc in range( - max(0, c - width // 2), min(image.shape[2], c + width // 2 + 1) - ): - if abs( - phases[z, sr, sc] - phases[z, r, c] - ) <= tolerance and np.isfinite(image[z, sr, sc]): - samples.append(image[z, sr, sc]) - if len(samples) >= minimum: - med[z, r, c] = np.median(samples) - first_med = med.copy() - for flat in selected: - z, r, c = np.unravel_index(flat, image.shape) - if ( - not np.isfinite(phases[z, r, c]) - or not np.isfinite(image[z, r, c]) - or np.isfinite(first_med[z, r, c]) - ): - continue - window = first_med[ - z, - max(0, r - height // 2) : r + height // 2 + 1, - max(0, c - width // 2) : c + width // 2 + 1, - ] - finite = window[np.isfinite(window)] - if finite.size: - med[z, r, c] = np.mean(finite) - return med - - -class TestPhaseMatchedFilter(unittest.TestCase): - """Check phase membership, spatial support, and frozen fallback medians.""" - - def test_overlapping_curved_bands_match_brute_force(self): - """Match explicit samples across depths, gaps, and chunk sizes.""" - rng = np.random.default_rng(8) - image = rng.normal(size=(2, 23, 17)).astype(np.float32) - # Independent, nonparallel trajectories with overlapping column ranges. - cols = np.arange(17) - ref_c = np.tile(cols, 4) - ref_r = np.r_[ - 4 + abs(cols - 8) // 2, - 16 - abs(cols - 8) // 2, - 3 + cols // 3, - 19 - cols // 3, - ].astype(float) - ref_d = np.repeat([0, 1], 34) - keep = ref_c != 7 # Column gap is not interpolated. - ref_r, ref_c, ref_d = ref_r[keep], ref_c[keep], ref_d[keep] - selected = np.flatnonzero(rng.random(image.size) > 0.15) - image.ravel()[selected[::29]] = np.nan - image.ravel()[selected[::41]] = np.inf - original = image.copy() - expected = brute_median( - image, selected, ref_r, ref_c, ref_d, 9, 7, 0, 3 - ) - for chunk_size in (1, 53, 4096): - actual = si.phase_matched_nanmedian( - image, - selected[::-1], - ref_r, - ref_c, - ref_d, - height=9, - width=7, - min_samples=3, - chunk_size=chunk_size, - ) - np.testing.assert_allclose( - actual, expected, atol=1e-6, equal_nan=True - ) - self.assertTrue(np.isnan(actual[:, :, 7]).all()) - np.testing.assert_array_equal(image, original) - - def test_signed_phase_not_absolute_distance(self): - """Keep opposite signed offsets in distinct sample populations.""" - image = np.zeros((1, 13, 11), dtype=np.float32) - cols = np.arange(11) - centers = np.full(11, 6) - image[0, 5, :] = 10 # phase -1 - image[0, 7, :] = 100 # phase +1 - selected = np.arange(image.size) - med = si.phase_matched_nanmedian(image, selected, centers, cols) - self.assertEqual(med[0, 5, 5], 10) - self.assertEqual(med[0, 7, 5], 100) - - def test_phase_tolerance_and_minimum_samples(self): - """Match the reference as tolerance and sample thresholds vary.""" - image = np.arange(5 * 7, dtype=float).reshape(1, 5, 7) - cols = np.arange(7) - centers = 2 + 0.25 * (cols % 2) - selected = np.arange(image.size) - for tolerance in (0, 0.3, 1.0): - for minimum in (1, 5, 100): - truth = brute_median( - image, - selected, - centers, - cols, - np.zeros(7), - 5, - 7, - tolerance, - minimum, - ) - result = si.phase_matched_nanmedian( - image, - selected, - centers, - cols, - height=5, - width=7, - phase_tolerance=tolerance, - min_samples=minimum, - ) - np.testing.assert_allclose(result, truth, equal_nan=True) - - def test_midpoint_rows_and_duplicate_centers_are_retained(self): - """Retain midpoint support while deduplicating reference centers.""" - image = np.ones((1, 16, 11)) - cols = np.tile(np.arange(11), 3) - centers = np.repeat([4, 4, 12], 11) # Deduplicate the top band. - selected = np.arange(image.size) - med = si.phase_matched_nanmedian(image, selected, centers, cols) - np.testing.assert_array_equal(med[0, 8], 1) # Equidistant bands. - self.assertTrue(np.isfinite(med[0, 4]).all()) - centers = np.repeat([4, 4, 13], 11) - med = si.phase_matched_nanmedian(image, selected, centers, cols) - np.testing.assert_array_equal( - med[0, 8:10], 1 - ) # Near ties also retained. - - def test_midpoint_tie_chooses_smaller_reference_row(self): - """Resolve equidistant references toward the smaller row.""" - image = np.array([[[10.0, 20.0, 30.0, 40.0, 50.0, 100.0, 200.0]]]) - cols = np.arange(7) - # First five pixels are ties with signed phase +4; last two are -4. - lower = np.r_[np.full(5, -4), np.full(2, -14)] - upper = np.full(7, 4) - selected = np.arange(image.size) - med = si.phase_matched_nanmedian( - image, - selected, - np.r_[lower, upper], - np.tile(cols, 2), - ) - self.assertEqual(med[0, 0, 3], 30) - - def test_default_exact_phase_does_not_pool_nearby_offsets(self): - """Pool nearby phases only when an explicit tolerance permits it.""" - image = np.full((1, 4, 11), np.nan) - cols = np.arange(11) - centers = np.where(cols % 2, 1.25, 1.0) - image[0, 2, :] = np.where(cols % 2, 100, 10) - selected = np.flatnonzero(np.isfinite(image)) - med = si.phase_matched_nanmedian(image, selected, centers, cols) - self.assertEqual(med[0, 2, 5], 100) # Five exact +0.75 matches. - self.assertEqual(med[0, 2, 4], 10) # Five exact +1 matches. - tolerant = si.phase_matched_nanmedian( - image, - selected, - centers, - cols, - phase_tolerance=0.5, - ) - self.assertEqual(tolerant[0, 2, 5], 10) - - def test_irregular_spacing_keeps_pixel_offsets_not_fractional_phase(self): - """Measure phase in pixels rather than local inter-band fractions.""" - image = np.full((1, 9, 7), np.nan) - cols = np.arange(7) - gaps = np.array([8, 12, 8, 12, 8, 12, 8]) - image[0, 3, :] = np.arange(1, 8) # +1 in every column. - refs = np.r_[np.full(7, 2), 2 + gaps] - med = si.phase_matched_nanmedian( - image, - np.flatnonzero(np.isfinite(image)), - refs, - np.tile(cols, 2), - ) - self.assertEqual(med[0, 3, 3], 4) - - def test_fallback_uses_first_pass_medians_not_raw_or_filled_values(self): - """Freeze first-pass medians so fallback values cannot propagate.""" - image = np.full((1, 3, 11), np.nan, dtype=np.float32) - image[0, 0] = np.arange(11) ** 2 - image[0, 1:3, 5] = 9999 - selected = np.flatnonzero(np.isfinite(image)) - cols = np.arange(11) - expected_med = [] - for col in cols: - samples = image[0, 0, max(0, col - 5) : col + 6] - median = np.median(samples) - expected_med.append(median) - for chunk in (1, 4, 4096): - med = si.phase_matched_nanmedian( - image, - selected[::-1], - np.zeros(11), - cols, - height=3, - chunk_size=chunk, - ) - np.testing.assert_allclose(med[0, 0], expected_med) - self.assertAlmostEqual( - float(med[0, 1, 5]), np.mean(expected_med), places=5 - ) - # Filled row 1 cannot propagate into row 2, even in later chunks. - self.assertTrue(np.isnan(med[0, 2, 5])) - self.assertTrue(np.isnan(med[~np.isfinite(image)]).all()) - ordered = si.phase_matched_nanmedian( - image, - selected, - np.zeros(11), - cols, - height=3, - chunk_size=chunk, - ) - np.testing.assert_allclose(ordered, med, equal_nan=True) - out = si.finalize_activity_image( - image.copy(), - selected, - np.zeros(selected.size), - np.zeros(11), - cols, - phase_window_height=3, - ) - expected = 9999 - np.mean(expected_med) - np.testing.assert_allclose(out[0, 1, 5], expected, rtol=1e-6) - self.assertEqual(out[0, 2, 5], 0) # Vertical window hits image edge. - - def test_empty_fallback_is_nan_without_warnings(self): - """Leave unsupported windows NaN without empty-reduction warnings.""" - import warnings - - image = np.ones((1, 3, 3)) - with warnings.catch_warnings(): - warnings.simplefilter("error", RuntimeWarning) - med = si.phase_matched_nanmedian( - image, - np.arange(image.size), - np.ones(3), - np.arange(3), - ) - self.assertTrue(np.isnan(med).all()) - - def test_missing_depth_geometry_and_empty_support(self): - """Require multi-plane geometry and preserve absent-plane NaNs.""" - image = np.ones((2, 9, 11)) - cols, centers = np.arange(11), np.full(11, 4) - selected = np.arange(image.size) - with self.assertRaisesRegex(ValueError, "ref_d"): - si.phase_matched_nanmedian(image, selected, centers, cols) - med = si.phase_matched_nanmedian( - image, selected, centers, cols, np.zeros(11) - ) - self.assertTrue(np.isfinite(med[0]).all()) - self.assertTrue(np.isnan(med[1]).all()) - for valid in (np.array([], dtype=int), selected): - med = si.phase_matched_nanmedian( - image * np.nan, valid, centers, cols, np.zeros(11) - ) - self.assertTrue(np.isnan(med).all()) - - def test_finalization_masks_samples_and_preserves_motion_geometry(self): - """Use shifted geometry and exclude invalid samples at finalization.""" - cols = np.arange(3, 14) - centers = 5 + abs(cols - 8) // 2 - image = np.arange(20 * 20, dtype=float).reshape(1, 20, 20) - selected = np.arange(image.size) - nan_ct = np.zeros(image.size) - nan_ct[::7] = 1 - dr, dc = 2, -1 - image_masked = image.copy() - image_masked.ravel()[nan_ct > 0.5] = np.nan - med = si.phase_matched_nanmedian( - image_masked, - selected[nan_ct <= 0.5], - centers + dr, - cols + dc, - ) - expected = image_masked - med - support = np.isfinite(image_masked) - for z, r, c in zip(*np.where(support)): - if ( - r < 3 - or r + 3 >= image.shape[1] - or not support[z, r - 3 : r + 4, c].all() - ): - expected[z, r, c] = 0 - actual = si.finalize_activity_image( - image.copy(), - selected, - nan_ct, - centers + dr, - cols + dc, - ) - np.testing.assert_allclose(actual, expected, equal_nan=True) - - def test_invalid_parameters(self): - """Reject invalid window, chunk, sample, and tolerance settings.""" - image = np.ones((1, 5, 7)) - for options in ( - {"height": 4}, - {"width": 0}, - {"chunk_size": 0}, - {"min_samples": 0}, - {"phase_tolerance": -1}, - {"phase_tolerance": np.inf}, - {"phase_tolerance": np.nan}, - ): - with self.subTest(options=options), self.assertRaises(ValueError): - si.phase_matched_nanmedian( - image, - np.arange(image.size), - np.full(7, 2), - np.arange(7), - **options, - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_bandsilo_pipeline.py b/tests/test_bandsilo_pipeline.py deleted file mode 100644 index cfc3016..0000000 --- a/tests/test_bandsilo_pipeline.py +++ /dev/null @@ -1,851 +0,0 @@ -"""Canonical band output assembly, shared writing and localization wiring. - -Tests target the implementation owners rather than compatibility re-exports. -Public driver composition is covered separately in test_band_workflow. -""" - -import os -import tempfile -import unittest -from contextlib import ExitStack -from unittest.mock import call, patch - -import h5py -import numpy as np -import torch - -from giant_python.extraction.band import result_assembly as pl -from giant_python.extraction.band import workflow as wf -from giant_python.extraction.band.types import ( - ResolvedAcquisition, - TrialTraceResult, -) -from giant_python.io.experiment_summary import write_summary -from giant_python.models.experiment_summary import ( - FRAME_INFO_KEYS, - ExperimentSummary, -) -from giant_python.models.params import ( - AnnotationOptions, - BandSiloParams, - ExecutionOptions, -) -from giant_python.numerics.baseline import assemble_dff - - -def _trial(n_frames, n_sources, n_channels, n_soma): - """One synthetic named trial record, matching the trace producer.""" - rng = np.random.default_rng(n_frames) - return TrialTraceResult( - rng.random((n_frames, n_sources)).astype(np.float32), # phi (dF_ls) - (rng.random((n_frames, n_sources)) + 5).astype(np.float32), # F0 - np.arange(n_frames, dtype=float), # frames - np.arange(3), # sel_pix_idxs - rng.random((n_frames, n_channels)).astype(np.float32), # global_f - (rng.random(n_frames), rng.random(n_frames), rng.random(n_frames)), - ( - np.zeros(n_frames, np.int16), - np.zeros(n_frames, np.int16), - np.zeros(n_frames, np.int16), - ), - rng.random((n_frames, n_soma, n_channels)).astype(np.float32), - ) - - -def _path_result( - n_sources=2, - n_channels=1, - nz=1, - npc=4, - npr=5, - draw_user_rois=False, - soma_masks=None, - soma_labels=None, - n_soma=0, -): - """Build a synthetic PathResult for one DMD path.""" - n_pixels = nz * npc * npr - results = [ - _trial(6, n_sources, n_channels, n_soma), - _trial(7, n_sources, n_channels, n_soma), - ] - return pl.PathResult( - a=torch.rand((n_pixels, n_sources)), - source_params=torch.rand((n_sources, 6)), - source_snr=np.arange(1, n_sources + 1, dtype=np.float32), - act_im=np.random.rand(nz, npc, npr).astype(np.float32), - mean_im=np.random.rand(n_channels, nz, npc, npr).astype(np.float32), - act_im_peaks=np.zeros((n_sources, 3)), - trace_results=results, - z_depths=np.zeros((nz, 1), dtype=np.int32), - num_fast_zs=nz, - dmd_pixels_per_column=npc, - dmd_pixels_per_row=npr, - num_channels=n_channels, - denoise_window=3, - baseline_window=5, - draw_user_rois=draw_user_rois, - soma_masks=soma_masks, - soma_labels=soma_labels, - yx_shape=(npc, npr), - ) - - -class TestLocalizeMedianMotion(unittest.TestCase): - """Check median geometry without running the expensive rho/NMF stages.""" - - def test_dog_settings_and_all_source_columns_reach_output(self): - """Keep DoG widths, reference centers and source matrices unchanged.""" - ref_d = np.array([0, 0]) - ref_r, ref_c = np.array([1, 2]), np.array([2, 2]) - selected = np.array([0, 1, 2]) - coords = np.zeros((3, 3)) - umyx = np.array([[2, -1]]) - mot_inds = np.zeros(4, dtype=int) - residual = np.ones((2, 4)) - rho = np.ones((3, 4)) - exclusion_mask = np.zeros((1, 3, 4), dtype=bool) - exclusion_mask[0, 0, 0] = True - seeds = np.array([[0, 1, 2], [0, 2, 2]]) - fitted = { - "A": torch.ones((12, 2)), - "source_params": torch.ones((2, 6)), - "source_snr": np.ones(2), - } - with ExitStack() as stack: - stack.enter_context( - patch.object( - wf.operators, "build_motion_h_matrices", return_value=["H"] - ) - ) - kernels = stack.enter_context( - patch.object( - wf.operators, - "gaussian_kernel_2d", - side_effect=[("center", (0, 0)), ("surround", (1, 1))], - ) - ) - stack.enter_context( - patch.object( - wf.geo, - "selected_pixels_2d_for_plane", - return_value=(selected, coords), - ) - ) - convolution = stack.enter_context( - patch.object( - wf.operators, - "build_convolution_matrix", - side_effect=["D", "D_exp"], - ) - ) - compute_rho = stack.enter_context( - patch.object(wf.activity, "compute_rho", return_value=rho) - ) - stack.enter_context( - patch.object( - wf.activity, "mask_high_nan_rho", return_value=np.zeros(3) - ) - ) - decay_kernel = stack.enter_context( - patch.object(wf.activity, "decay_kernel_1d") - ) - smooth = stack.enter_context( - patch.object(wf.activity, "smooth_rho", return_value=rho) - ) - stack.enter_context( - patch.object(wf.si, "accumulate_activity_image") - ) - stack.enter_context(patch.object(wf.si, "finalize_activity_image")) - detect = stack.enter_context( - patch.object(wf, "get_act_im_peaks", return_value=seeds) - ) - fit = stack.enter_context( - patch.object(wf, "fit_sources", return_value=fitted) - ) - result = wf._localize( - residual, - umyx, - mot_inds, - selected, - coords, - 12, - ref_d, - ref_r, - ref_c, - 1, - 3, - 4, - np.ones((3, 3)), - 2, - None, - None, - 10, - BandSiloParams(), - execution=ExecutionOptions(verbose=False), - exclusion_mask=exclusion_mask, - ) - self.assertIs(detect.call_args.kwargs["exclusion_mask"], exclusion_mask) - self.assertEqual(kernels.call_args_list, [call(0.9), call(4.5)]) - self.assertEqual( - convolution.call_args_list[0].args[1:], ("center", (0, 0)) - ) - self.assertEqual( - convolution.call_args_list[1].args[1:], ("surround", (1, 1)) - ) - self.assertIs(compute_rho.call_args.args[7], ref_d) - self.assertIs(compute_rho.call_args.args[8], ref_r) - self.assertIs(compute_rho.call_args.args[9], ref_c) - self.assertIs(fit.call_args.args[0], seeds) - self.assertIs(fit.call_args.args[1], residual) - self.assertIs(fit.call_args.args[3], umyx) - self.assertIs(fit.call_args.args[4], mot_inds) - self.assertIs(fit.call_args.args[5], selected) - self.assertIs(fit.call_args.args[6], coords) - decay_kernel.assert_called_once_with(BandSiloParams().decay_tau_s, 10) - self.assertIs(smooth.call_args.args[1], decay_kernel.return_value) - self.assertIs( - fit.call_args.kwargs["temporal_kernel"], decay_kernel.return_value - ) - self.assertIs(result[2], fitted["A"]) - self.assertIs(result[3], fitted["source_params"]) - self.assertIs(result[4], fitted["source_snr"]) - np.testing.assert_array_equal(result[1], seeds) - self.assertIsNot(result[1], seeds) - - def test_dominant_retained_motion_shifts_only_median_geometry(self): - """Dropped frames do not vote; default run policy preserves centers.""" - ref_r = np.array([10, 19]) - ref_c = np.array([8, 8]) - # Dropped frames outnumber either retained bin and must be ignored. - mot_inds = np.array([-1, -1, -1, -1, 0, 1, 1, 1]) - for displacement in ((2, -5), (-3, 4), (0, 0)): - with self.subTest(displacement=displacement), ExitStack() as stack: - umyx = np.array([[7, 9], displacement], dtype=float) - for owner, name in ( - (wf.operators, "build_motion_h_matrices"), - (wf.operators, "gaussian_kernel_2d"), - (wf.geo, "selected_pixels_2d_for_plane"), - (wf.operators, "build_convolution_matrix"), - (wf.activity, "compute_rho"), - (wf.activity, "mask_high_nan_rho"), - (wf.activity, "decay_kernel_1d"), - (wf.activity, "smooth_rho"), - ): - stack.enter_context( - patch.object(owner, name, return_value=(None, None)) - ) - stack.enter_context( - patch.object(wf.si, "accumulate_activity_image") - ) - # Stop at finalization; peak detection and NMF are unrelated. - finalize = stack.enter_context( - patch.object( - wf.si, - "finalize_activity_image", - side_effect=RuntimeError("median reached"), - ) - ) - with self.assertRaisesRegex(RuntimeError, "median reached"): - wf._localize( - residual=np.zeros((2, mot_inds.size)), - umyx=umyx, - mot_inds_yx=mot_inds, - sel_pix_idxs=np.array([0]), - pixel_coords=None, - n_pixels=1600, - ref_d=np.array([0, 0]), - ref_r=ref_r, - ref_c=ref_c, - num_fast_zs=1, - dmd_pixels_per_column=40, - dmd_pixels_per_row=40, - psf2d=np.ones((3, 3)), - num_super_pixels=2, - sparse_h_inds=None, - sparse_h_vals=None, - align_hz=10, - params=BandSiloParams(), - ) - finalize.assert_called_once() - np.testing.assert_array_equal( - finalize.call_args.kwargs["ref_d"], [0, 0] - ) - np.testing.assert_array_equal( - finalize.call_args.kwargs["ref_r"], ref_r + displacement[0] - ) - np.testing.assert_array_equal( - finalize.call_args.kwargs["ref_c"], ref_c + displacement[1] - ) - np.testing.assert_array_equal(ref_r, [10, 19]) - np.testing.assert_array_equal(ref_c, [8, 8]) - - -class TestStageComposition(unittest.TestCase): - """Check background wiring and empty localization without sparse fits.""" - - def test_persisted_background_interpolation_is_cubic(self): - """Output metadata records the fixed background algorithm.""" - metadata = wf._params_dict( - BandSiloParams(), - ResolvedAcquisition(num_channels=1, align_hz=100.0), - AnnotationOptions(), - ) - self.assertEqual(metadata["backgroundInterpolation"], "cubic") - - def test_persisted_trace_settings_exclude_removed_mode_keys(self): - """Default metadata retains tuning settings but no trace opt-outs.""" - params = BandSiloParams() - metadata = wf._params_dict( - params, - ResolvedAcquisition(num_channels=1, align_hz=100.0), - AnnotationOptions(), - ) - for name in ("simple_trace_extraction", "convolution_trace_extraction"): - self.assertNotIn(name, metadata) - self.assertEqual(metadata["decayTau_s"], params.decay_tau_s) - for name in ( - "trace_beta_fraction", "trace_max_iter", "trace_tolerance", - "trace_debias", "trace_event_threshold", "random_seed", - ): - with self.subTest(name=name): - self.assertEqual(metadata[name], getattr(params, name)) - - def test_default_execution_skips_empty_planes_and_preserves_grid(self): - """Interpolate without PSF expansion; preserve the background grid.""" - data = np.array([[3.0, 4.0], [5.0, 6.0]], dtype=np.float32) - selected = np.array([4, 5]) - ref_d, ref_r, ref_c = ( - np.array([1, 1]), - np.array([0, 1]), - np.array([0, 1]), - ) - umyx, mot_inds = np.array([[0, 0]]), np.array([0, 0]) - plane_pixels = np.array([[0, 0], [0, 1]]) - interpolated = data + 10 - baseline, assembled = data + 20, data + 30 - params = BandSiloParams(baseline_window_s=0.5) - with ExitStack() as stack: - expand = stack.enter_context( - patch.object(wf.operators, "expand_psf") - ) - planes = stack.enter_context( - patch.object( - wf.geo, - "selected_pixels_2d_for_plane", - side_effect=[ - (np.array([], dtype=int), np.empty((0, 2))), - (np.array([0, 1]), plane_pixels), - ], - ) - ) - interpolate = stack.enter_context( - patch.object( - wf.bg, - "build_interp_data", - return_value=(interpolated, None, None), - ) - ) - rolling = stack.enter_context( - patch.object( - wf.bg, "compute_rolling_baseline", return_value=baseline - ) - ) - assemble = stack.enter_context( - patch.object( - wf.bg, "assemble_background", return_value=assembled - ) - ) - result = wf._estimate_background( - data, - umyx, - mot_inds, - selected, - ref_d, - ref_r, - ref_c, - 2, - 2, - 2, - 10, - params, - ) - expand.assert_not_called() - self.assertEqual(planes.call_count, 2) - interpolate.assert_called_once() - np.testing.assert_array_equal(interpolate.call_args.args[0], data) - np.testing.assert_array_equal(interpolate.call_args.args[1], ref_r) - np.testing.assert_array_equal(interpolate.call_args.args[2], ref_c) - self.assertIs(interpolate.call_args.args[3], plane_pixels) - self.assertEqual( - interpolate.call_args.kwargs["method"], - "cubic", - ) - rolling.assert_called_once() - np.testing.assert_array_equal(rolling.call_args.args[0], interpolated) - self.assertEqual(rolling.call_args.args[0].dtype, np.float32) - self.assertEqual(rolling.call_args.args[1], 5) - assemble.assert_called_once() - self.assertIs(assemble.call_args.args[0], baseline) - self.assertIs(assemble.call_args.args[1], umyx) - self.assertIs(assemble.call_args.args[2], mot_inds) - self.assertEqual(assemble.call_args.args[7:], (2, 2, 2, 2)) - self.assertIs(result, assembled) - - def test_no_retained_motion_bins_do_not_invent_a_source_fit(self): - """Characterize the existing empty argmax error without a fallback.""" - with ExitStack() as stack: - for owner, name in ( - (wf.operators, "build_motion_h_matrices"), - (wf.activity, "compute_rho"), - (wf.activity, "mask_high_nan_rho"), - (wf.activity, "smooth_rho"), - (wf.si, "accumulate_activity_image"), - ): - stack.enter_context(patch.object(owner, name)) - finalize = stack.enter_context( - patch.object(wf.si, "finalize_activity_image") - ) - fit = stack.enter_context(patch.object(wf, "fit_sources")) - with self.assertRaisesRegex(ValueError, "argmax.*empty"): - wf._localize( - np.ones((2, 3)), - np.empty((0, 2)), - np.full(3, -1), - np.array([], dtype=int), - np.empty((0, 3)), - 4, - np.array([0, 0]), - np.array([0, 1]), - np.array([0, 1]), - 0, - 2, - 2, - np.ones((3, 3)), - 2, - None, - None, - 10, - BandSiloParams(), - ) - finalize.assert_not_called() - fit.assert_not_called() - - -class TestDmdUserRois(unittest.TestCase): - """_dmd_user_rois slices a resolved session selection per DMD.""" - - def test_none_is_neutral(self): - """No session selection yields (None, None, []).""" - self.assertEqual(pl._dmd_user_rois(None, "DMD1"), (None, None, [])) - - def test_slices_key(self): - """The named DMD's masks, labels, and superpixels are returned.""" - sps = [np.array([0, 1])] - user_rois = { - "user_roi_masks": {"DMD1": ["mask"]}, - "user_roi_labels": {"DMD1": ["roi"]}, - "user_roi_superpixels": {"DMD1": sps}, - } - masks, labels, out_sps = pl._dmd_user_rois(user_rois, "DMD1") - self.assertEqual(masks, ["mask"]) - self.assertEqual(labels, ["roi"]) - self.assertIs(out_sps, sps) - - def test_missing_key_defaults(self): - """A DMD absent from the selection yields empty superpixels.""" - user_rois = { - "user_roi_masks": {}, - "user_roi_labels": {}, - "user_roi_superpixels": {}, - } - masks, labels, out_sps = pl._dmd_user_rois(user_rois, "DMD2") - self.assertIsNone(masks) - self.assertIsNone(labels) - self.assertEqual(out_sps, []) - - -class TestWithNanChannel(unittest.TestCase): - """_with_nan_channel reshapes and NaN-fills non-activity channels.""" - - def test_shape_and_channels(self): - """(frames, sources) -> (sources, channels, frames); ch>0 is NaN.""" - arr = np.arange(6, dtype=np.float32).reshape(3, 2) # 3 frames, 2 src - out = pl._with_nan_channel(arr, 2) - self.assertEqual(out.shape, (2, 2, 3)) - np.testing.assert_array_equal(out[:, 0, :], arr.T) - self.assertTrue(np.all(np.isnan(out[:, 1, :]))) - - -class TestAssemblePathSummary(unittest.TestCase): - """Typed path assembly preserves numerical shapes and content.""" - - def test_shapes(self): - """Every dataset has the schema shape.""" - pr = _path_result(n_sources=2, n_channels=1, nz=1, npc=4, npr=5) - path = pl.assemble_path_summary(pr) - self.assertEqual(len(path.sources), 2) - for source in path.sources: - self.assertEqual(source.profile.shape, (1, 4, 5)) - self.assertEqual(source.coords.shape, (3,)) - self.assertEqual(source.df_ls.shape, (1, 13)) - self.assertEqual(source.f0.shape, (1, 13)) - for values in (source.df_denoised, source.events): - self.assertEqual(values.shape, source.df_ls.shape) - self.assertTrue(np.isnan(values).all()) - self.assertEqual(source.df_ls.dtype, np.float64) - self.assertEqual(source.f0.dtype, np.float64) - np.testing.assert_array_equal([s.snr for s in path.sources], [1, 2]) - self.assertEqual(path.global_f.shape, (1, 13)) - self.assertEqual( - list(path.frame_info.trial_num_frames.ravel()), [6, 7] - ) - self.assertEqual(path.frame_info.discard_frames.shape, (13, 1)) - self.assertEqual(path.user_rois, []) - self.assertFalse(path.annotation_enabled) - self.assertIs(path.z_depths, pr.z_depths) - - def test_two_channels_global_shape(self): - """Two channels widen the global-F channel axis.""" - pr = _path_result(n_channels=2) - path = pl.assemble_path_summary(pr) - self.assertEqual(path.global_f.shape, (2, 13)) - for field in ("df_ls", "f0", "df_denoised", "events"): - values = getattr(path.sources[0], field) - self.assertEqual(values.shape, (2, 13)) - self.assertTrue(np.isnan(values[1]).all()) - - def test_residual_backprojection_enters_final_f0(self): - """F = dF_ls + F0_ls + residual before the final baseline estimate.""" - pr = _path_result() - raw_ls = np.concatenate([r.d_f for r in pr.trace_results]) - raw_f0 = np.concatenate([r.f0_ls for r in pr.trace_results]) - pr.trace_results = [ - r._replace(f_residual=np.full_like(r.d_f, 0.25 * (i + 1))) - for i, r in enumerate(pr.trace_results) - ] - residual = np.concatenate([r.f_residual for r in pr.trace_results]) - _, expected_f0, expected_df, _ = assemble_dff( - raw_ls, raw_f0 + residual, pr.denoise_window, pr.baseline_window - ) - path = pl.assemble_path_summary(pr) - np.testing.assert_array_equal(path.sources[0].f0[0], expected_f0[:, 0]) - np.testing.assert_array_equal(path.sources[0].df_ls[0], expected_df[:, 0]) - - def test_independent_traces_stitch_skips_masks_and_legacy_trials(self): - """Only LS gets baseline processing; raw fit outputs keep trial order.""" - for channels in (1, 2): - with self.subTest(channels=channels): - pr = _path_result(n_sources=2, n_channels=channels) - fitted = [] - for index, trial in enumerate(pr.trace_results): - signal = trial.d_f.copy() - 0.5 - signal[1] = np.nan - denoised = ( - np.arange(signal.size, dtype=np.float32) - .reshape(signal.shape) + 100 * (index + 1) - ) - events = np.zeros_like(signal) - events[2, index] = 7 + index - denoised[1] = np.nan - events[1] = np.nan - # Missing transient values must not alter the LS mask. - denoised[3, 0] = np.nan - events[4, 1] = np.nan - fitted.append(trial._replace( - d_f=signal, d_f_denoised=denoised, events=events, - frame_line_idxs=trial.frame_line_idxs + 100 * index, - )) - skipped = _trial(0, 2, channels, 0)._replace( - d_f_denoised=np.empty((0, 2), dtype=np.float32), - events=np.empty((0, 2), dtype=np.float32), - ) - legacy = _trial(5, 2, channels, 0) - pr.trace_results = [fitted[0], skipped, fitted[1], legacy] - snapshots = [ - tuple(None if value is None else value.copy() for value in ( - trial.d_f, trial.f0_ls, trial.d_f_denoised, trial.events - )) - for trial in pr.trace_results - ] - raw_ls = np.concatenate([r.d_f for r in pr.trace_results]) - raw_f0 = np.concatenate([r.f0_ls for r in pr.trace_results]) - full, expected_f0, expected_ls, _ = assemble_dff( - raw_ls, raw_f0, pr.denoise_window, pr.baseline_window - ) - expected_denoised = np.concatenate([ - fitted[0].d_f_denoised, fitted[1].d_f_denoised, - np.full_like(legacy.d_f, np.nan), - ]) - expected_events = np.concatenate([ - fitted[0].events, fitted[1].events, - np.full_like(legacy.d_f, np.nan), - ]) - with patch.object(pl, "assemble_dff", wraps=assemble_dff) as baseline: - path = pl.assemble_path_summary(pr) - baseline.assert_called_once() - np.testing.assert_array_equal(baseline.call_args.args[0], raw_ls) - np.testing.assert_array_equal(baseline.call_args.args[1], raw_f0) - self.assertEqual(baseline.call_args.args[2:], (3, 5)) - self.assertFalse(np.allclose(raw_ls, expected_ls, equal_nan=True)) - for index, source in enumerate(path.sources): - for field, expected in ( - ("df_ls", expected_ls), ("f0", expected_f0), - ("df_denoised", expected_denoised), - ("events", expected_events), - ): - actual = getattr(source, field) - self.assertEqual(actual.shape, (channels, 18)) - np.testing.assert_array_equal(actual[0], expected[:, index]) - self.assertTrue(np.isnan(actual[1:]).all()) - self.assertEqual(source.df_denoised.dtype, np.float32) - self.assertEqual(source.events.dtype, np.float32) - np.testing.assert_array_equal( - path.frame_info.trial_num_frames.ravel(), [6, 0, 7, 5] - ) - np.testing.assert_array_equal( - path.frame_info.discard_frames.ravel(), - np.any(np.isnan(full), axis=1), - ) - self.assertFalse(path.frame_info.discard_frames[[3, 4, 9, 10]].any()) - np.testing.assert_array_equal( - path.frame_info.frame_line_idxs.ravel(), - np.concatenate([r.frame_line_idxs for r in pr.trace_results]), - ) - np.testing.assert_array_equal( - path.global_f, - np.concatenate([r.global_f for r in pr.trace_results]).T, - ) - for trial, before in zip(pr.trace_results, snapshots): - for actual, expected in zip( - (trial.d_f, trial.f0_ls, trial.d_f_denoised, trial.events), - before, - ): - if expected is None: - self.assertIsNone(actual) - else: - np.testing.assert_array_equal(actual, expected) - - # Changing only the fit cannot change LS, F0 or discard flags. - pr.trace_results = [r._replace( - d_f_denoised=(None if r.d_f_denoised is None - else r.d_f_denoised + 500), - events=(None if r.events is None else r.events * 3), - ) for r in pr.trace_results] - changed = pl.assemble_path_summary(pr) - for before, after in zip(path.sources, changed.sources): - np.testing.assert_array_equal(after.df_ls, before.df_ls) - np.testing.assert_array_equal(after.f0, before.f0) - np.testing.assert_array_equal( - after.df_denoised[0], before.df_denoised[0] + 500 - ) - np.testing.assert_array_equal(after.events[0], before.events[0] * 3) - np.testing.assert_array_equal( - changed.frame_info.discard_frames, path.frame_info.discard_frames - ) - - def test_missing_optional_fields_are_filled_independently(self): - """One absent fit field never erases or substitutes the other field.""" - pr = _path_result(n_channels=2) - first, second = pr.trace_results - events = np.full_like(first.d_f, 7) - denoised = np.full_like(second.d_f, 50) - pr.trace_results = [ - first._replace(events=events), - second._replace(d_f_denoised=denoised), - ] - path = pl.assemble_path_summary(pr) - for index, source in enumerate(path.sources): - self.assertTrue(np.isnan(source.df_denoised[0, :6]).all()) - np.testing.assert_array_equal(source.df_denoised[0, 6:], denoised[:, index]) - np.testing.assert_array_equal(source.events[0, :6], events[:, index]) - self.assertTrue(np.isnan(source.events[0, 6:]).all()) - self.assertTrue(np.isnan(source.df_denoised[1]).all()) - self.assertTrue(np.isnan(source.events[1]).all()) - - def test_user_rois_assembled(self): - """draw_user_rois with masks assembles the user-ROI datasets.""" - masks = [np.ones((1, 4, 5), dtype=bool), np.zeros((1, 4, 5), bool)] - pr = _path_result( - draw_user_rois=True, - soma_masks=masks, - soma_labels=["SOMA", "SOMA2"], - n_soma=2, - ) - path = pl.assemble_path_summary(pr) - self.assertTrue(path.annotation_enabled) - self.assertEqual(len(path.user_rois), 2) - for index, roi in enumerate(path.user_rois): - np.testing.assert_array_equal(roi.mask, masks[index]) - self.assertEqual(roi.f.shape, (1, 13)) - self.assertEqual(roi.f.dtype, np.float32) - np.testing.assert_array_equal( - roi.f, - np.concatenate( - [r.user_roi_f[:, index] for r in pr.trace_results] - ).T, - ) - - def test_user_rois_empty_masklist(self): - """draw_user_rois with an empty mask list yields a 0-ROI mask array.""" - pr = _path_result( - draw_user_rois=True, soma_masks=[], soma_labels=[], n_soma=0 - ) - path = pl.assemble_path_summary(pr) - self.assertTrue(path.annotation_enabled) - self.assertEqual(path.user_rois, []) - self.assertEqual(path._empty_roi_arrays["mask"].shape, (0, 1, 4, 5)) - self.assertEqual(path._empty_roi_arrays["F"].shape, (0, 1, 13)) - - def test_empty_sources_preserve_geometry_and_frame_data(self): - """No sources still produces shaped empty tensors and real globals.""" - pr = _path_result(n_sources=0, n_channels=2) - path = pl.assemble_path_summary(pr) - self.assertEqual(path.sources, []) - self.assertEqual( - path._empty_source_arrays["profiles"].shape, (0, 1, 4, 5) - ) - self.assertEqual(path._empty_source_arrays["dF_ls"].shape, (0, 2, 13)) - for key in ("dF_denoised", "events"): - self.assertEqual(path._empty_source_arrays[key].shape, (0, 2, 13)) - self.assertEqual(path._empty_source_arrays[key].dtype, np.float32) - self.assertEqual(path._empty_source_arrays["SNR"].dtype, np.float32) - self.assertEqual(path.global_f.shape, (2, 13)) - self.assertFalse(path.frame_info.discard_frames.any()) - - -class TestWriteExperimentSummary(unittest.TestCase): - """write_summary produces the MATLAB-compatible schema.""" - - def _write_and_read(self, assembled_paths, params=None): - """Write then reopen, returning the h5 file handle's data as dict.""" - params = params or {"operator": "test", "numChannels": 1} - path = tempfile.mktemp(suffix=".h5") - write_summary( - ExperimentSummary(paths=assembled_paths, params=params), path - ) - return path - - def test_schema_and_values(self): - """Root flag/params + one Path group with the expected datasets.""" - asm = pl.assemble_path_summary(_path_result()) - path = self._write_and_read([asm]) - try: - with h5py.File(path, "r") as f: - self.assertEqual(int(f["row_major"][()]), 1) - self.assertIn("params", f) - grp = f["Path1"] - self.assertEqual( - grp["sources/spatial/profiles"].shape, (2, 1, 4, 5) - ) - self.assertEqual( - grp["sources/temporal/dF_ls"].shape, (2, 1, 13) - ) - self.assertEqual(grp["global/F"].shape, (1, 13)) - np.testing.assert_array_equal(grp["Z_depths"], asm.z_depths) - self.assertNotIn("z_depths", grp) - for key in FRAME_INFO_KEYS: - self.assertIn(key, grp["frame_info"]) - self.assertIn("act_im", grp["visualizations"]) - for key in ("dF_denoised", "events"): - values = grp[f"sources/temporal/{key}"][()] - self.assertEqual(values.shape, (2, 1, 13)) - self.assertTrue(np.isnan(values).all()) - rois = grp["user_rois"] - self.assertFalse(rois.attrs["annotation_enabled"]) - self.assertEqual(set(rois), {"mask", "labels", "F", "Fsvd"}) - self.assertEqual(rois["mask"].shape, (0, 1, 4, 5)) - self.assertEqual(rois["labels"].shape, (0, 1)) - self.assertEqual(rois["F"].shape, (0, 1, 13)) - self.assertEqual(rois["Fsvd"].shape, (0, 1, 13)) - finally: - os.remove(path) - - def test_user_rois_written(self): - """The user_rois group is written when soma ROIs are present.""" - masks = [np.ones((1, 4, 5), dtype=bool)] - pr = _path_result( - draw_user_rois=True, - soma_masks=masks, - soma_labels=["SOMA"], - n_soma=1, - ) - path = self._write_and_read([pl.assemble_path_summary(pr)]) - try: - with h5py.File(path, "r") as f: - ur = f["Path1/user_rois"] - self.assertEqual(ur["mask"].shape, (1, 1, 4, 5)) - self.assertEqual(ur["F"].shape, (1, 1, 13)) - self.assertEqual(ur["labels"].shape, (1, 1)) - self.assertTrue(ur.attrs["annotation_enabled"]) - self.assertEqual(ur["Fsvd"].shape, ur["F"].shape) - self.assertTrue(np.isnan(ur["Fsvd"][()]).all()) - finally: - os.remove(path) - - def test_multi_path_and_overwrite(self): - """Two paths write Path1/Path2; re-writing overwrites in place.""" - asm = [pl.assemble_path_summary(_path_result(), i) for i in range(2)] - path = self._write_and_read(asm) - try: - # second write to the same file overwrites the Path groups - write_summary( - ExperimentSummary(paths=asm, params={"operator": "test"}), path - ) - with h5py.File(path, "r") as f: - self.assertIn("Path1", f) - self.assertIn("Path2", f) - finally: - os.remove(path) - - -class TestExperimentSummary(unittest.TestCase): - """Construct summaries directly from numerical path models.""" - - def test_all_path_fields_are_retained(self): - """Paths retain distinct images, frame metadata and globals.""" - assembled = [ - pl.assemble_path_summary(_path_result(), i) for i in range(2) - ] - assembled[0].z_depths = np.array([[10]]) - assembled[1].z_depths = np.array([[20]]) - assembled[0].global_f[:] = 3 - assembled[1].global_f[:] = 7 - summary = ExperimentSummary(paths=assembled, params={}) - self.assertEqual(len(summary.paths), 2) - for index, path in enumerate(summary.paths): - self.assertEqual(path.path_index, index) - np.testing.assert_array_equal(path.z_depths, [[10 + 10 * index]]) - np.testing.assert_array_equal(path.global_f, 3 + 4 * index) - self.assertIs(path, assembled[index]) - - def test_sources_and_visualizations(self): - """Per-path sources are populated with profile/coords/traces/SNR.""" - asm = pl.assemble_path_summary(_path_result(n_sources=2)) - summary = ExperimentSummary(paths=[asm], params={"operator": "t"}) - self.assertEqual(len(summary.paths), 1) - self.assertEqual(len(summary.paths[0].sources), 2) - src = summary.paths[0].sources[0] - self.assertEqual(src.profile.shape, (1, 4, 5)) - self.assertEqual(src.snr, 1.0) - self.assertIsNotNone(summary.paths[0].visualizations.act_im) - self.assertEqual(summary.paths[0].user_rois, []) - self.assertIsNotNone(summary.paths[0].frame_info.trial_num_frames) - - def test_user_rois_populated(self): - """User ROIs appear in the summary when present.""" - masks = [np.ones((1, 4, 5), dtype=bool)] - pr = _path_result( - draw_user_rois=True, - soma_masks=masks, - soma_labels=["SOMA"], - n_soma=1, - ) - asm = pl.assemble_path_summary(pr) - summary = ExperimentSummary(paths=[asm], params={"operator": "t"}) - self.assertEqual(len(summary.paths[0].user_rois), 1) - self.assertEqual(summary.paths[0].user_rois[0].label, "SOMA") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_bandsilo_progress.py b/tests/test_bandsilo_progress.py deleted file mode 100644 index 056c47c..0000000 --- a/tests/test_bandsilo_progress.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Tests for giant_python.progress (opt-in logging + bars).""" - -import contextlib -import io -import unittest - -from giant_python.progress import log, progress - - -class TestLog(unittest.TestCase): - """log prints only when verbose is set.""" - - def test_verbose_prints(self): - """A verbose log writes the message to stdout.""" - buf = io.StringIO() - with contextlib.redirect_stdout(buf): - log("hello", True) - self.assertEqual(buf.getvalue().strip(), "hello") - - def test_quiet_is_silent(self): - """A non-verbose log writes nothing.""" - buf = io.StringIO() - with contextlib.redirect_stdout(buf): - log("hello", False) - self.assertEqual(buf.getvalue(), "") - - -class TestProgress(unittest.TestCase): - """progress wraps an iterable only when verbose is set.""" - - def test_quiet_returns_same_object(self): - """Without verbose, the iterable is returned unchanged.""" - r = range(3) - self.assertIs(progress(r, desc="x", verbose=False), r) - - def test_verbose_wraps_but_preserves_items(self): - """With verbose, a tqdm wrapper still yields the same items.""" - out = progress(range(3), desc="x", verbose=True) - self.assertIsNot(out, range(3)) - self.assertEqual(list(out), [0, 1, 2]) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_bandsilo_summary_images.py b/tests/test_bandsilo_summary_images.py deleted file mode 100644 index 1e48781..0000000 --- a/tests/test_bandsilo_summary_images.py +++ /dev/null @@ -1,441 +0,0 @@ -"""Tests for giant_python.extraction.band.summary_images (Phase 4). - -Exercises the mean image, the spatio-temporal local-maxima activity-image -accumulation, and the median-subtraction finalize on small synthetic inputs. -""" - -import unittest -from unittest.mock import patch - -import numpy as np - -from giant_python.extraction.band import summary_images as si - - -class TestMeanImage(unittest.TestCase): - """compute_mean_image single- and dual-channel.""" - - def test_single_channel(self): - """Channel-1 mean lands at the shifted reference pixels.""" - ref_d = np.array([0, 0]) - ref_r = np.array([1, 2]) - ref_c = np.array([1, 3]) - low = np.array([[1.0, 3.0], [2.0, 4.0]], dtype=np.float32) - unique_motion = np.array([[0, 0, 0]], dtype=float) - mot_inds = np.array([0, 0]) - frames = np.array([True, True]) - mean_im = si.compute_mean_image( - low, - unique_motion, - mot_inds, - frames, - ref_d, - ref_r, - ref_c, - num_fast_zs=1, - dmd_pixels_per_column=5, - dmd_pixels_per_row=5, - num_channels=1, - ) - self.assertEqual(mean_im.shape, (1, 1, 5, 5)) - self.assertAlmostEqual(float(mean_im[0, 0, 1, 1]), 2.0, places=5) - self.assertAlmostEqual(float(mean_im[0, 0, 2, 3]), 3.0, places=5) - - def test_dual_channel_fills_second(self): - """Channel-2 mean is written when num_channels >= 2.""" - ref_d = np.array([0]) - ref_r = np.array([1]) - ref_c = np.array([1]) - low = np.array([[1.0, 3.0]], dtype=np.float32) - low2 = np.array([[10.0, 30.0]], dtype=np.float32) - unique_motion = np.array([[0, 0, 0]], dtype=float) - mot_inds = np.array([0, 0]) - frames = np.array([True, True]) - mean_im = si.compute_mean_image( - low, - unique_motion, - mot_inds, - frames, - ref_d, - ref_r, - ref_c, - num_fast_zs=1, - dmd_pixels_per_column=4, - dmd_pixels_per_row=4, - num_channels=2, - low_res_data2_norm=low2, - ) - self.assertEqual(mean_im.shape, (2, 1, 4, 4)) - self.assertAlmostEqual(float(mean_im[1, 0, 1, 1]), 20.0, places=5) - - -class TestAccumulateActivityImage(unittest.TestCase): - """accumulate_activity_image local-maxima detection and accumulation.""" - - def test_single_spike_is_detected(self): - """One bright voxel exceeding its neighbors accumulates its square.""" - num_fast_zs, npc, npr = 1, 7, 7 - n_pix = npc * npr - sel_pix_idxs = np.arange(n_pix) - n_frames = 5 - rho = np.zeros((n_pix, n_frames), dtype=np.float32) - # spike at center pixel (row 3, col 3), frame 2, value 4 -> 16 - center_flat = 3 * npr + 3 - rho[center_flat, 2] = 4.0 - act_im = si.accumulate_activity_image( - rho, sel_pix_idxs, num_fast_zs, npc, npr - ) - self.assertEqual(act_im.shape, (num_fast_zs, npc, npr)) - self.assertAlmostEqual(float(act_im[0, 3, 3]), 16.0, places=4) - # neighbors did not accumulate - self.assertAlmostEqual(float(act_im[0, 3, 4]), 0.0, places=5) - - def test_nan_neighbor_suppresses_peak(self): - """A NaN adjacent to a candidate peak removes it via dilation.""" - num_fast_zs, npc, npr = 1, 7, 7 - n_pix = npc * npr - sel_pix_idxs = np.arange(n_pix) - n_frames = 5 - rho = np.zeros((n_pix, n_frames), dtype=np.float32) - center_flat = 3 * npr + 3 - rho[center_flat, 2] = 4.0 - rho[3 * npr + 4, 2] = np.nan # NaN right next to the peak - act_im = si.accumulate_activity_image( - rho, sel_pix_idxs, num_fast_zs, npc, npr - ) - self.assertAlmostEqual(float(act_im[0, 3, 3]), 0.0, places=5) - - def test_small_batches_cover_multiple_iterations(self): - """A tiny batch_size forces multiple batches, same output.""" - num_fast_zs, npc, npr = 1, 7, 7 - n_pix = npc * npr - sel_pix_idxs = np.arange(n_pix) - rho = np.zeros((n_pix, 8), dtype=np.float32) - rho[3 * npr + 3, 4] = 3.0 - big = si.accumulate_activity_image( - rho, sel_pix_idxs, num_fast_zs, npc, npr, batch_size=1000 - ) - small = si.accumulate_activity_image( - rho, sel_pix_idxs, num_fast_zs, npc, npr, batch_size=2 - ) - np.testing.assert_allclose(big, small, rtol=1e-5) - - def test_too_few_frames_yields_zero_image(self): - """A batch with <= 2 frames has no temporal interior and accumulates - nothing.""" - num_fast_zs, npc, npr = 1, 7, 7 - n_pix = npc * npr - sel_pix_idxs = np.arange(n_pix) - rho = np.ones((n_pix, 1), dtype=np.float32) # single frame - act_im = si.accumulate_activity_image( - rho, sel_pix_idxs, num_fast_zs, npc, npr - ) - self.assertTrue(np.all(act_im == 0.0)) - - -class TestFinalizeActivityImage(unittest.TestCase): - """Activity-image masking and local median subtraction in raw units.""" - - def test_vertical_window_requires_full_valid_support(self): - """Suppress windows crossing edges or invalid vertical support.""" - image = np.ones((2, 19, 13), dtype=np.float32) - selected_mask = np.ones(image.shape, dtype=bool) - selected_mask[0, :4] = False - selected_mask[0, 15:] = False - selected = np.flatnonzero(selected_mask) - nan_ct = np.zeros(selected.size) - dropped = np.ravel_multi_index((0, 9, 6), image.shape) - nan_ct[selected == dropped] = 1 - image[1, 9, 7] = np.nan - image[1, 9, 8] = np.inf - support = selected_mask & np.isfinite(image) - support[0, 9, 6] = False - for height in (1, 3, 7): - geometry = dict( - ref_r=np.array([9, 9]), - ref_c=np.array([6, 6]), - ref_d=np.array([0, 1]), - phase_window_height=height, - ) - # A fixed median isolates final edge suppression. - with patch.object( - si, - "phase_matched_nanmedian", - return_value=np.zeros_like(image), - ): - out = si.finalize_activity_image( - image.copy(), - selected, - nan_ct, - **geometry, - ) - for z, r, c in zip(*np.where(support)): - lo, hi = r - height // 2, r + height // 2 + 1 - fits = ( - lo >= 0 - and hi <= image.shape[1] - and support[z, lo:hi, c].all() - ) - self.assertEqual(out[z, r, c], 1 if fits else 0) - self.assertTrue(np.isnan(out[~selected_mask]).all()) - self.assertTrue(np.isnan(out[0, 9, 6])) - self.assertTrue(np.isnan(out[1, 9, 7])) - self.assertEqual(out[1, 9, 0], 1) # Horizontal edges survive. - - def test_median_subtraction_preserves_mask(self): - """Subtract the median without restoring excluded pixels.""" - image = np.ones((1, 13, 17), dtype=np.float32) - image[0, 6, 8] = 7 - selected = np.arange(1, image.size) - nan_ct = np.zeros(selected.size) - nan_ct[-1] = 0.9 - for geometry in ( - {}, - {"ref_r": np.full(17, 6), "ref_c": np.arange(17)}, - ): - with self.subTest(snake=bool(geometry)): - out = si.finalize_activity_image( - image.copy(), - selected, - nan_ct, - **geometry, - ) - self.assertEqual(out[0, 6, 8], 6) - self.assertEqual(out[0, 6, 7], 0) - self.assertTrue(np.isnan(out.ravel()[0])) - self.assertTrue(np.isnan(out.ravel()[-1])) - expected = np.zeros_like(image) - expected[0, 6, 8] = 6 - expected.ravel()[[0, -1]] = np.nan - np.testing.assert_allclose(out, expected, equal_nan=True) - - def test_constant_neighborhoods_are_zero_and_spike_is_preserved(self): - """Subtract constant backgrounds without masking valid neighbors.""" - num_fast_zs, npc, npr = 1, 13, 13 - act_im = np.ones((num_fast_zs, npc, npr), dtype=np.float32) - # all pixels selected and valid (nan_ct = 0) - sel_pix_idxs = np.arange(npc * npr) - nan_ct = np.zeros(sel_pix_idxs.size) - for geometry in ( - {}, - {"ref_r": np.full(npr, 6), "ref_c": np.arange(npr)}, - ): - for spike in (0, 10): - with self.subTest(geometry=bool(geometry), spike=spike): - image = act_im.copy() - image[0, 6, 6] += spike - out = si.finalize_activity_image( - image, sel_pix_idxs, nan_ct, **geometry - ) - radius = 3 if geometry else 2 - self.assertTrue(np.all(out[:, :radius] == 0)) - self.assertTrue(np.all(out[:, -radius:] == 0)) - expected = np.zeros_like(act_im) - expected[0, 6, 6] = spike - np.testing.assert_array_equal(out, expected) - - def test_high_nan_pixels_masked(self): - """Pixels with nan_ct > 0.5 are excluded and set to NaN.""" - npc, npr = 13, 13 - act_im = np.arange(npc * npr, dtype=np.float32).reshape(1, npc, npr) - rows, cols = np.meshgrid( - np.arange(2, 5), np.arange(2, 8), indexing="ij" - ) - sel_pix_idxs = (rows * npr + cols).ravel() - nan_ct = np.zeros(sel_pix_idxs.size) - nan_ct[sel_pix_idxs == 3 * npr + 4] = 0.9 - out = si.finalize_activity_image(act_im, sel_pix_idxs, nan_ct) - # the dropped pixel and all unselected pixels are NaN - self.assertTrue(np.isnan(out[0, 3, 4])) - self.assertTrue(np.isnan(out[0, 0, 0])) - self.assertFalse(np.isnan(out[0, 3, 3])) - - def test_median_follows_curved_snake(self): - """The 3x11 footprint follows reference-center row changes.""" - size = 17 - cols = np.arange(size) - center_rows = np.abs(cols - size // 2) + 3 - act_im = np.zeros((1, size, size), dtype=np.float32) - for col, row in zip(cols, center_rows): - act_im[0, row - 1 : row + 2, col] = 10.0 - - sel_pix_idxs = np.arange(size * size) - med = si.snake_aligned_nanmedian( - act_im, - sel_pix_idxs, - ref_r=center_rows, - ref_c=cols, - height=3, - ) - - center = size // 2 - self.assertAlmostEqual(float(med[0, 3, center]), 10.0, places=5) - - def test_median_follows_motion_shifted_bend(self): - """Shifted geometry follows the bend in output-image coordinates.""" - cols = np.arange(8, 25) - center_rows = np.abs(cols - 16) + 8 - for dr, dc in ((2, 5), (-2, -5), (0, 0)): - with self.subTest(dr=dr, dc=dc): - act_im = np.zeros((1, 40, 40), dtype=np.float32) - for col, row in zip(cols + dc, center_rows + dr): - act_im[0, row - 1 : row + 2, col] = 10.0 - sel_pix_idxs = np.arange(act_im.size) - med = si.snake_aligned_nanmedian( - act_im, - sel_pix_idxs, - ref_r=center_rows + dr, - ref_c=cols + dc, - ) - self.assertAlmostEqual(float(med[0, 8 + dr, 16 + dc]), 10.0) - if dc: - unshifted = si.snake_aligned_nanmedian( - act_im, - sel_pix_idxs, - ref_r=center_rows, - ref_c=cols, - ) - self.assertLess(float(unshifted[0, 8 + dr, 16 + dc]), 10.0) - - def test_trajectory_does_not_jump_when_outer_snake_is_missing(self): - """Changing snake count does not move the shared trajectory.""" - size = 40 - cols = np.arange(11) - ref_c = np.repeat(cols, 5) - ref_r = np.tile(np.arange(1, 38, 9), cols.size) - keep = ~((ref_c >= 5) & (ref_r == 1)) - - act_im = np.zeros((1, size, cols.size), dtype=np.float32) - act_im[0, 18:21, :] = 10.0 - sel_pix_idxs = np.arange(act_im.size) - med = si.snake_aligned_nanmedian( - act_im, - sel_pix_idxs, - ref_r=ref_r[keep], - ref_c=ref_c[keep], - ) - - self.assertAlmostEqual(float(med[0, 19, 4]), 10.0, places=5) - - def test_median_subtraction_preserves_raw_units_and_scaling(self): - """Both paths subtract footprint medians and retain input scaling.""" - image = ( - np.broadcast_to(np.arange(17), (1, 17, 17)) - .astype(np.float32) - .copy() - ) - image[0, 8, 8] += 6 - selected = np.arange(image.size) - for geometry, height in ( - ({}, 5), - ({"ref_r": np.full(17, 8), "ref_c": np.arange(17)}, 1), - ): - with self.subTest(height=height): - samples = image[0, 8 - height // 2 : 9 + height // 2, 3:14] - median = np.nanmedian(samples) - out = si.finalize_activity_image( - image.copy(), selected, np.zeros(selected.size), **geometry - ) - self.assertAlmostEqual( - float(out[0, 8, 8]), float(image[0, 8, 8] - median) - ) - rescaled = si.finalize_activity_image( - 4 * image + 13, - selected, - np.zeros(selected.size), - **geometry, - ) - np.testing.assert_allclose(4 * out, rescaled, equal_nan=True) - - def test_median_subtraction_follows_curved_footprint(self): - """Subtract the phase-matched median at a curved band center.""" - cols = np.arange(17) - ref_r = np.abs(cols - 8) + 5 - image = np.zeros((1, 25, 17), dtype=np.float32) - for col, row in zip(cols, ref_r): - image[0, row - 2 : row + 3, col] = ( - 20 + col - 8 + 100 * np.arange(-2, 3) - ) - image[0, 5, 8] = 25 - out = si.finalize_activity_image( - image, - np.arange(image.size), - np.zeros(image.size), - ref_r=ref_r, - ref_c=cols, - ) - # The default 7-row window reaches center-phase samples in columns - # 5:12 at this bend. Replacing 20 with 25 gives median=21. - self.assertAlmostEqual(float(out[0, 5, 8]), 4.0, places=6) - - -class TestSnakeMedian(unittest.TestCase): - """Medians use the exact curved, clipped, section-limited footprint.""" - - def test_matches_explicit_samples_across_chunks(self): - """Match clipped trajectory samples independently of chunk size.""" - rng = np.random.default_rng(42) - image = rng.normal(size=(2, 20, 17)).astype(np.float32) - image[:, 4, ::3] = np.nan - cols = np.r_[np.arange(7), np.arange(10, 17)] - ref_r = np.abs(cols - 8) + 3 - points = [(0, 3, 0), (0, 4, 6), (1, 10, 10), (1, 19, 16)] - selected = np.ravel_multi_index(np.array(points).T, image.shape) - original = image.copy() - for height in (3, 5): - med = si.snake_aligned_nanmedian( - image, - selected, - ref_r, - cols, - height=height, - chunk_size=2, - ) - for depth, row, col in points: - samples = [] - for sample_col in range(max(0, col - 5), min(17, col + 6)): - # Nearest reference endpoint extends into the gap; - # ties select the left section (columns 0 through 8). - if (sample_col <= 8) != (col <= 8): - continue - nearest_col = cols[np.argmin(np.abs(cols - sample_col))] - delta = abs(nearest_col - 8) - abs(col - 8) - for offset in range(-(height // 2), height // 2 + 1): - sample_row = row + delta + offset - if 0 <= sample_row < image.shape[1]: - samples.append( - image[depth, sample_row, sample_col] - ) - expected_med = np.nanmedian(samples) - self.assertAlmostEqual( - float(med[depth, row, col]), float(expected_med) - ) - unselected = np.ones(image.size, dtype=bool) - unselected[selected] = False - self.assertTrue(np.all(np.isnan(med.ravel()[unselected]))) - for chunk_size in (1, 4096): - other_chunk = si.snake_aligned_nanmedian( - image, - selected, - ref_r, - cols, - height=height, - chunk_size=chunk_size, - ) - np.testing.assert_allclose(med, other_chunk, equal_nan=True) - np.testing.assert_array_equal(image, original) - - def test_empty_and_all_nan_support(self): - """Preserve NaNs for empty output selections and missing samples.""" - image = np.full((1, 5, 11), np.nan, dtype=np.float32) - cols = np.arange(11) - for selected in (np.array([], dtype=int), np.array([27])): - with self.subTest(selected=selected.size): - med = si.snake_aligned_nanmedian( - image, selected, np.full(11, 2), cols - ) - self.assertTrue(np.all(np.isnan(med))) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_bandsilo_traces.py b/tests/test_bandsilo_traces.py deleted file mode 100644 index de1553e..0000000 --- a/tests/test_bandsilo_traces.py +++ /dev/null @@ -1,665 +0,0 @@ -"""Tests for canonical band traces (Phase 7 high-res traces). - -Covers the per-trial compute (motion interp/binning, per-motion least-squares -``phi``/``F0``, global and per-ROI fluorescence) and the IO wrapper's -skip/monkeypatched-read paths. ``compute_high_res_traces`` is additionally -cross-checked against a verbatim copy of the reference in the project's -development notes; here we assert structural behavior on synthetic data. -""" - -import unittest -from unittest.mock import patch - -import numpy as np -import torch - -from giant_python.extraction.band import operators -from giant_python.extraction.band import traces as tr -from giant_python.extraction.band.types import TrialTraceResult -from giant_python.extraction.band.transient import TransientOptions, TransientResult - - -def _geometry(npc=15, npr=15, num_fast_zs=2): - """Build a small consistent geometry (subsample map + sparse H + PSF).""" - plane = npc * npr - positions = [(r, c) for r in range(4, 11) for c in range(4, 11)] - rows = [] - for z in range(num_fast_zs): - for r, c in positions: - rows.append([z * plane + r * npr + c, len(rows) + 1]) - smi = np.array(rows, dtype=np.int32) - yy, xx = np.mgrid[-2:3, -2:3] - psf2d = np.exp(-(yy**2 + xx**2) / 2.0).astype(np.float32) - sh_inds, sh_vals = operators.build_sparse_h(smi, psf2d, npc, npr) - return dict( - npc=npc, - npr=npr, - nz=num_fast_zs, - nsp=smi.shape[0], - smi=smi, - psf2d=psf2d, - sh_inds=sh_inds, - sh_vals=sh_vals, - ) - - -def _trial_inputs(num_channels=1, unique_motion_ds=None, seed=2): - """Synthetic loaded-trial arrays + geometry for compute_high_res_traces.""" - g = _geometry() - rng = np.random.default_rng(seed) - nsp = g["nsp"] - n_ds = 60 - n_frames = 40 - ds_frames = np.arange(n_ds, dtype=float) - frames = np.linspace(1, n_ds - 2, n_frames) - a_data = dict( - DSframes=ds_frames, - motionDSr=(np.arange(n_ds) % 2).astype(float), # bins 0 and 1 - motionDSc=np.zeros(n_ds), - motionDSz=np.zeros(n_ds), - onlineYshift=rng.standard_normal(n_ds), - onlineXshift=rng.standard_normal(n_ds), - onlineZshift=rng.standard_normal(n_ds), - ) - data = rng.standard_normal((nsp, n_frames)).astype(np.float32) + 2 - data2 = ( - rng.standard_normal((nsp, n_frames)).astype(np.float32) + 1 - if num_channels >= 2 - else None - ) - background_ds = rng.standard_normal((nsp, n_ds)).astype(np.float32) + 1 - if unique_motion_ds is None: - unique_motion_ds = np.array([[0, 0], [1, 0]], dtype=float) - mot_keep_ds = np.arange(unique_motion_ds.shape[0]) - n_pixels = g["nz"] * g["npc"] * g["npr"] - a_final = torch.rand((n_pixels, 3), dtype=torch.float32) - soma_sps = [np.array([0, 1, 2]), np.array([5, 6, 7, 8])] - return dict( - g=g, - data=data, - data2=data2, - background_ds=background_ds, - frames=frames, - a_data=a_data, - a_final=a_final, - unique_motion_ds=unique_motion_ds, - mot_keep_ds=mot_keep_ds, - soma_sps=soma_sps, - num_channels=num_channels, - n_frames=n_frames, - ) - - -def _compute(inp, **kwargs): - """Call compute_high_res_traces with the assembled inputs dict.""" - g = inp["g"] - return tr.compute_high_res_traces( - inp["data"], - inp["data2"], - inp["background_ds"], - inp["frames"], - inp["a_data"], - g["smi"], - g["sh_inds"], - g["sh_vals"], - inp["a_final"], - inp["unique_motion_ds"], - inp["mot_keep_ds"], - 0.0, - g["psf2d"], - g["nsp"], - g["nz"], - g["npc"], - g["npr"], - inp["num_channels"], - inp["soma_sps"], - **kwargs, - ) - - -class TestTrialTraceResult(unittest.TestCase): - """The original eight positional fields retain their identity and order.""" - - def test_legacy_constructor_defaults_and_extended_fields(self): - original = _compute(_trial_inputs()) - legacy = TrialTraceResult(*original[:8]) - self.assertEqual(len(legacy), 11) - self.assertEqual(legacy._fields, ( - "d_f", "f0_ls", "frame_line_idxs", "selected_pixels", "global_f", - "reference_offsets", "online_shifts", "user_roi_f", - "d_f_denoised", "events", "f_residual", - )) - self.assertIsNone(legacy.d_f_denoised) - self.assertIsNone(legacy.events) - denoised = np.full_like(legacy.d_f, 13) - events = np.full_like(legacy.d_f, 2) - partial = TrialTraceResult(*legacy[:8], denoised) - self.assertIs(partial.d_f_denoised, denoised) - self.assertIsNone(partial.events) - extended = TrialTraceResult(*legacy[:8], denoised, events) - for index, value in enumerate(original[:8]): - self.assertIs(legacy[index], value) - self.assertIs(extended[index], value) - self.assertIs(extended.d_f_denoised, denoised) - self.assertIs(extended.events, events) - self.assertIs(extended[8], denoised) - self.assertIs(extended[9], events) - - -class TestWeightedTraceAverage(unittest.TestCase): - """Simple traces use profile intensities, not a regression coefficient.""" - - def test_normalized_intensities_and_scale_invariance(self): - """Overlapping and duplicate profiles are averaged independently.""" - profiles = torch.tensor([[1., 0., 7.], [3., 2., 21.], [0., 2., 0.]]) - data = torch.tensor([[2., -2.], [6., 4.], [10., 8.]]) - expected = np.array([[5., 8., 5.], [2.5, 6., 2.5]]) - actual = tr._weighted_trace_average(profiles, data) - np.testing.assert_allclose(actual, expected) - np.testing.assert_allclose( - tr._weighted_trace_average(profiles * 0.01, data), - expected, rtol=1e-6, - ) - - def test_uniform_zero_and_empty_profiles(self): - """Uniform weights give the mean; no support is NaN, not zero.""" - data = torch.tensor([[2., 4.], [6., 8.]]) - profiles = torch.tensor([[3., 0.], [3., 0.]]) - result = tr._weighted_trace_average(profiles, data) - np.testing.assert_allclose(result[:, 0], [4., 6.]) - self.assertTrue(torch.isnan(result[:, 1]).all()) - self.assertEqual( - tr._weighted_trace_average(torch.empty(2, 0), data).shape, - (2, 0), - ) - - def test_motion_projection_background_and_dropped_frames(self): - """Normalize after motion projection and use identical dF/F0 weights.""" - profiles = torch.tensor([[1., 2.], [3., 0.], [0., 4.]]) - operators_by_motion = [ - torch.eye(3).to_sparse(), - torch.tensor([[0., 2., 0.], [0., 0., 1.], [0., 0., 0.]]).to_sparse(), - ] - data = np.array([[2., 4., 8., 1.], [6., 8., 2., 2.], - [10., 12., 4., 3.]], dtype=np.float32) - background = np.array([[1., 1., 2., 0.], [2., 2., 1., 0.], - [3., 3., 3., 0.]], dtype=np.float32) - bins = np.array([0, 0, 1, -1]) - args = ( - data, background, np.array([[0, 0], [1, 0]]), bins, - np.array([0, 1]), np.empty((0, 2)), np.empty(0), np.arange(3), - profiles, 3, 3, - ) - with patch.object( - tr, "build_motion_h_matrices", return_value=operators_by_motion - ), patch.object( - tr, "solve_phi_motion", side_effect=AssertionError("LS called") - ): - phi, f0 = tr._solve_trial_phi_f0( - *args, simple_trace_extraction=True - ) - for motion, h in enumerate(operators_by_motion): - selected = np.flatnonzero(bins == motion) - projected = (h.to_dense() @ profiles).numpy() - for source in range(profiles.shape[1]): - weights = projected[:, source] - np.testing.assert_allclose( - phi[selected, source], - np.average((data - background)[:, selected], axis=0, - weights=weights), rtol=1e-6, - ) - np.testing.assert_allclose( - f0[selected, source], - np.average(background[:, selected], axis=0, - weights=weights), rtol=1e-6, - ) - np.testing.assert_allclose( - (phi + f0)[selected, source], - np.average(data[:, selected], axis=0, weights=weights), - rtol=1e-6, - ) - self.assertTrue(torch.isnan(phi[-1]).all()) - self.assertTrue(torch.isnan(f0[-1]).all()) - - -class TestLeastSquaresTraceExtraction(unittest.TestCase): - """Missing high-resolution observations must not abort a trial.""" - - def test_transient_outputs_never_replace_qr_and_default_return_stays_pair(self): - """Distinct fit payloads stay separate, including their missing masks.""" - profiles = torch.eye(2) - h_mots = [torch.eye(2).to_sparse(), (2 * torch.eye(2)).to_sparse()] - data = np.array([ - [-4., 2., 10., 4., 5., np.inf, 14., 8.], - [3., np.nan, 6., 8., 1., 3., 10., 4.], - ], dtype=np.float32) - background = np.ones_like(data) - background[0, 3] = np.nan - bins = np.array([0, 0, 1, 1, -1, 0, 1, 0]) - valid = np.array([True, False, True, False, False, False, True, True]) - denoised = np.arange(16, dtype=np.float32).reshape(8, 2) + 100 - denoised[~valid] = np.nan - latent_events = np.arange(16, dtype=np.float32).reshape(8, 2) + 20 - latent_events[0] = 0 # Initial state is not a new event. - expected_events = latent_events.copy() - expected_events[~valid] = np.nan - args = ( - data, background, np.array([[0, 0], [1, 0]]), bins, - np.array([0, 1]), np.empty((0, 2)), np.empty(0), np.arange(2), - profiles, 2, 2, - ) - options = TransientOptions() - - def fitted(*args, **kwargs): - """Return fresh arrays so masking cannot affect later calls.""" - return TransientResult( - d_f=denoised.copy(), events=latent_events.copy(), - initial_state=np.array([100., 101.]), - model_trace=np.full((8, 2), 999., dtype=np.float32), - metadata={"converged": True}, - ) - - with patch.object(tr, "build_motion_h_matrices", return_value=h_mots): - qr = tr._solve_trial_phi_f0(*args) - no_fit = tr._solve_trial_phi_f0(*args, return_transient=True) - with patch.object(tr, "fit_transient_traces", side_effect=fitted) as fit: - pair = tr._solve_trial_phi_f0( - *args, transient_options=options, analyze_hz=50, - ) - extended = tr._solve_trial_phi_f0( - *args, transient_options=options, analyze_hz=50, - return_transient=True, - ) - self.assertEqual(len(qr), 2) - self.assertEqual(len(pair), 2) - self.assertEqual(len(extended), 5) - self.assertEqual(len(no_fit), 5) - self.assertEqual(fit.call_count, 2) - for result in (pair, extended, no_fit): - for actual, expected in zip(result[:2], qr): - torch.testing.assert_close(actual, expected, equal_nan=True) - np.testing.assert_array_equal(qr[0][0], [-5., 2.]) - np.testing.assert_array_equal(qr[0][2], [4.5, 2.5]) - np.testing.assert_array_equal(qr[1][[1, 5]], np.ones((2, 2))) - np.testing.assert_array_equal(extended[2], denoised) - np.testing.assert_array_equal(extended[3], expected_events) - self.assertTrue(torch.isnan(no_fit[2]).all()) - self.assertTrue(torch.isnan(no_fit[3]).all()) - for call in fit.call_args_list: - np.testing.assert_array_equal(call.args[0], data - background) - np.testing.assert_array_equal(call.args[1][0], np.eye(2)) - np.testing.assert_array_equal(call.args[1][1], 2 * np.eye(2)) - np.testing.assert_array_equal(call.args[2], bins) - np.testing.assert_array_equal(call.args[3], valid) - self.assertEqual(call.args[4], 50) - self.assertIs(call.kwargs["options"], options) - - def test_missing_data_and_background_in_motion_bins(self): - """dF and F0 retain their own missing-frame masks across motion bins.""" - profiles = torch.tensor([[1., 2.], [3., 0.], [0., 4.]]) - h_mots = [ - torch.eye(3).to_sparse(), - torch.diag(torch.tensor([2., 1., 3.])).to_sparse(), - ] - data = np.arange(24, dtype=np.float32).reshape(3, 8) + 5 - background = np.ones_like(data) - bins = np.array([0, 0, 0, 1, 1, 1, 1, -1]) - args = ( - data, background, np.array([[0, 0], [1, 0]]), bins, - np.array([0, 1]), np.empty((0, 2)), np.empty(0), np.arange(3), - profiles, 3, 3, - ) - with patch.object(tr, "build_motion_h_matrices", return_value=h_mots): - expected_phi, expected_f0 = tr._solve_trial_phi_f0(*args) - data[0, 1] = np.nan - data[:, 2] = np.nan - background[1, 3] = np.nan - data[2, 4] = np.inf - phi, f0 = tr._solve_trial_phi_f0(*args) - - torch.testing.assert_close(phi[[0, 5, 6]], expected_phi[[0, 5, 6]]) - self.assertTrue(torch.isnan(phi[[1, 2, 3, 4, 7]]).all()) - valid_f0 = [0, 1, 2, 4, 5, 6] - torch.testing.assert_close(f0[valid_f0], expected_f0[valid_f0]) - self.assertTrue(torch.isnan(f0[[3, 7]]).all()) - - def test_f0_and_residual_are_profile_backprojections(self): - """F0 is not de-mixed; the residual backprojects data - bg - x phi.""" - profiles = torch.tensor([[1., 2.], [3., 0.], [1., 4.]]) - h_mots = [torch.eye(3).to_sparse()] - rng = np.random.default_rng(0) - data = rng.uniform(5, 10, (3, 6)).astype(np.float32) - background = rng.uniform(1, 3, (3, 6)).astype(np.float32) - args = ( - data, background, np.array([[0, 0]]), np.zeros(6, dtype=int), - np.array([0]), np.empty((0, 2)), np.empty(0), np.arange(3), - profiles, 3, 3, - ) - with patch.object(tr, "build_motion_h_matrices", return_value=h_mots): - phi, f0, _, _, f_res = tr._solve_trial_phi_f0( - *args, return_transient=True - ) - x = profiles.numpy() - norm = np.sum(x ** 2, axis=0) - np.testing.assert_allclose(f0, background.T @ x / norm, rtol=1e-5) - expected_res = (data - background - x @ phi.numpy().T).T @ x / norm - np.testing.assert_allclose(f_res, expected_res, atol=1e-5) - lstsq_f0 = np.linalg.lstsq(x, background, rcond=None)[0].T - self.assertFalse(np.allclose(f0, lstsq_f0, rtol=1e-3)) - - -class TestBinTrialMotion(unittest.TestCase): - """The configured z tolerance controls trial-frame retention.""" - - def test_z_tolerance_and_inclusive_boundaries(self): - motion_z = np.array([-3, -2, -1, 0, 1, 2, 3], dtype=float) - args = ( - np.zeros(7), np.zeros(7), motion_z, 0.5, - np.array([[0, 0]]), np.array([0]), - ) - default = tr._bin_trial_motion(*args) - np.testing.assert_array_equal( - default[3], np.abs(motion_z - 0.5) <= 1.5 - ) - for tolerance in (0.0, 0.5, 2.5, 4.0): - with self.subTest(z_tol=tolerance): - _, mot_inds, _, kept = tr._bin_trial_motion( - *args, z_tol=tolerance - ) - expected = np.abs(motion_z - 0.5) <= tolerance - np.testing.assert_array_equal(kept, expected) - np.testing.assert_array_equal(mot_inds[~expected], -1) - - -class TestComputeHighResTraces(unittest.TestCase): - """compute_high_res_traces per-trial numerics.""" - - def test_transient_preserves_qr_masks_geometry_and_auxiliary_outputs(self): - """A real fit adds denoised/events without replacing any legacy field.""" - inp = _trial_inputs(num_channels=2) - inp["a_data"]["motionDSz"][20:30] = 5 - inp["data"][0, 4] = np.nan - inp["data"][1, 5] = np.inf - inp["background_ds"][0, 0:4] = np.nan - inp["frames"] = np.ceil(inp["frames"]) - baseline = _compute(inp) - spatial_before = inp["a_final"].clone() - options = TransientOptions(max_iter=1500) - fitted = [] - original_fit = tr.fit_transient_traces - - def fit_trial(*args, **kwargs): - """Capture the real fit before the wrapper masks exposed events.""" - result = original_fit(*args, **kwargs) - fitted.append(result) - return result - - with patch.object(tr, "fit_transient_traces", side_effect=fit_trial) as solve: - out = _compute(inp, transient_options=options, analyze_hz=50) - solve.assert_called_once() - self.assertEqual(solve.call_args.args[4], 50) - self.assertIs(solve.call_args.kwargs["options"], options) - self.assertEqual(len(out), 11) - for actual, expected in zip(out[:8], baseline[:8]): - np.testing.assert_array_equal(actual, expected) - np.testing.assert_array_equal( - out.d_f_denoised, fitted[0].d_f.astype(np.float32) - ) - expected_events = fitted[0].events.astype(np.float32) - expected_events[~solve.call_args.args[3]] = np.nan - np.testing.assert_array_equal(out.events, expected_events) - missing = np.isnan(baseline.d_f) - self.assertTrue(missing.any()) - self.assertTrue((~missing).any()) - for values in (out.d_f_denoised, out.events): - self.assertEqual(values.shape, baseline.d_f.shape) - self.assertEqual(values.dtype, np.float32) - np.testing.assert_array_equal(np.isnan(values), missing) - self.assertTrue((values[~missing] >= 0).all()) - self.assertFalse(np.shares_memory(values, out.d_f)) - self.assertFalse(np.shares_memory(out.d_f_denoised, out.events)) - self.assertFalse(np.allclose(out.d_f, out.d_f_denoised, equal_nan=True)) - torch.testing.assert_close(inp["a_final"], spatial_before) - - def test_transient_all_missing_no_sources_and_failed_fit(self): - """Empty trials/sources are safe; failed fits never become normal output.""" - inp = _trial_inputs() - options = TransientOptions() - inp["data"][:] = np.nan - with patch.object(tr, "fit_transient_traces") as solve: - out = _compute(inp, transient_options=options, analyze_hz=50) - solve.assert_not_called() - self.assertTrue(np.isnan(out.d_f).all()) - self.assertTrue(np.isnan(out.d_f_denoised).all()) - self.assertTrue(np.isnan(out.events).all()) - inp["a_final"] = inp["a_final"][:, :0] - out = _compute(inp, transient_options=options, analyze_hz=50) - self.assertEqual(out.d_f.shape, (40, 0)) - self.assertEqual(out.d_f_denoised.shape, (40, 0)) - self.assertEqual(out.events.shape, (40, 0)) - inp = _trial_inputs() - with self.assertWarns(RuntimeWarning), self.assertRaisesRegex(RuntimeError, "did not converge"): - _compute(inp, transient_options=TransientOptions(max_iter=1), analyze_hz=50) - - def test_transient_requires_explicit_rate_and_ordered_frames(self): - """Line positions must not be interpreted as time in seconds.""" - inp = _trial_inputs() - options = TransientOptions() - with self.assertRaisesRegex(ValueError, "analyze_hz"): - _compute(inp, transient_options=options) - with self.assertRaisesRegex(ValueError, "mutually exclusive"): - _compute(inp, transient_options=options, analyze_hz=50, - simple_trace_extraction=True) - inp["frames"] = inp["frames"][::-1] - with self.assertRaisesRegex(ValueError, "chronological"): - _compute(inp, transient_options=options, analyze_hz=50) - - def test_default_remains_least_squares(self): - """Isolated low-level extraction still permits QR without a fit.""" - inp = _trial_inputs() - with ( - patch.object(tr, "solve_phi_motion", wraps=tr.solve_phi_motion) as ls, - patch.object(tr, "fit_transient_traces") as fit, - ): - default = _compute(inp) - fit.assert_not_called() - self.assertGreater(ls.call_count, 0) - explicit = _compute(inp, simple_trace_extraction=False) - np.testing.assert_array_equal(default.d_f, explicit.d_f) - np.testing.assert_array_equal(default.f0_ls, explicit.f0_ls) - self.assertTrue(np.isnan(default.d_f_denoised).all()) - self.assertTrue(np.isnan(default.events).all()) - - def test_simple_mode_preserves_mask_and_auxiliary_outputs(self): - """Only dF/F0 extraction changes; z rejection and other channels stay.""" - inp = _trial_inputs(num_channels=2) - inp["a_data"]["motionDSz"][20:40] = 5 - least_squares = _compute(inp) - with patch.object( - tr, "solve_phi_motion", side_effect=AssertionError("LS called") - ): - simple = _compute(inp, simple_trace_extraction=True) - self.assertTrue(np.isnan(simple.d_f).any()) - self.assertTrue(np.isfinite(simple.d_f).any()) - for a, b in ((simple.d_f, least_squares.d_f), - (simple.f0_ls, least_squares.f0_ls)): - self.assertEqual(a.shape, b.shape) - np.testing.assert_array_equal(np.isnan(a), np.isnan(b)) - for field in ("global_f", "user_roi_f", "frame_line_idxs", - "selected_pixels", "reference_offsets", "online_shifts"): - np.testing.assert_array_equal( - getattr(simple, field), getattr(least_squares, field) - ) - - def test_single_channel_shapes_and_fit(self): - """Returns ten fields; the original eight retain their positions.""" - inp = _trial_inputs(num_channels=1) - out = _compute(inp) - self.assertIsInstance(out, TrialTraceResult) - self.assertIs(out.d_f, out[0]) - self.assertIs(out.f0_ls, out[1]) - self.assertIs(out.frame_line_idxs, out[2]) - self.assertIs(out.selected_pixels, out[3]) - self.assertIs(out.global_f, out[4]) - self.assertIs(out.reference_offsets, out[5]) - self.assertIs(out.online_shifts, out[6]) - self.assertIs(out.user_roi_f, out[7]) - self.assertIs(out.d_f_denoised, out[8]) - self.assertIs(out.events, out[9]) - self.assertIs(out.f_residual, out[10]) - ( - phi, f0, frames, sel, global_f, motion, online, f_soma, - denoised, events, f_residual, - ) = out - n_sources = inp["a_final"].shape[1] - self.assertEqual(phi.shape, (inp["n_frames"], n_sources)) - self.assertEqual(f0.shape, (inp["n_frames"], n_sources)) - self.assertEqual(f_residual.shape, phi.shape) - for values in (denoised, events): - self.assertEqual(values.shape, phi.shape) - self.assertTrue(np.isnan(values).all()) - self.assertEqual(frames.shape, (inp["n_frames"],)) - self.assertEqual(global_f.shape, (inp["n_frames"], 1)) - self.assertEqual( - f_soma.shape, (inp["n_frames"], len(inp["soma_sps"]), 1) - ) - self.assertEqual(len(motion), 3) - self.assertEqual(len(online), 3) - self.assertTrue(np.any(np.isfinite(phi))) - - def test_two_channels(self): - """A second channel adds columns to global and soma fluorescence.""" - inp = _trial_inputs(num_channels=2) - out = _compute(inp) - global_f, f_soma = out.global_f, out.user_roi_f - self.assertEqual(global_f.shape, (inp["n_frames"], 2)) - self.assertEqual( - f_soma.shape, (inp["n_frames"], len(inp["soma_sps"]), 2) - ) - # both channels populated where frames are kept - self.assertTrue(np.any(np.isfinite(f_soma[:, :, 1]))) - - def test_unmatched_ds_bin_skipped(self): - """A low-res motion bin absent from the trial is skipped.""" - umd = np.array([[0, 0], [1, 0], [5, 5]], dtype=float) # [5,5] absent - inp = _trial_inputs(num_channels=1, unique_motion_ds=umd) - out = _compute(inp) - # still produces finite fits for the two present bins - self.assertTrue(np.any(np.isfinite(out[0]))) - - -class TestGetHighResTraces(unittest.TestCase): - """get_high_res_traces IO wrapper (skip + monkeypatched read).""" - - def test_skipped_trial_returns_empty(self): - """keep_trial=False returns all eleven empty fields without reading.""" - loader = patch.object( - tr, "_load_high_res_trial_data", side_effect=AssertionError("read skipped trial") - ) - fit = patch.object( - tr, "fit_transient_traces", side_effect=AssertionError("fit skipped trial") - ) - loader.start() - fit.start() - self.addCleanup(loader.stop) - self.addCleanup(fit.stop) - inp = _trial_inputs(num_channels=2) - g = inp["g"] - out = tr.get_high_res_traces( - (0, False, inp["background_ds"]), - 0, - 100.0, - np.zeros((g["nsp"], 1), dtype=np.int32), - "unused", - {}, - g["smi"], - g["sh_inds"], - g["sh_vals"], - inp["a_final"].numpy(), - inp["unique_motion_ds"], - inp["mot_keep_ds"], - 0.0, - g["psf2d"], - g["nsp"], - g["nz"], - g["npc"], - g["npr"], - 2, - inp["soma_sps"], - transient_options=TransientOptions(), - ) - self.assertIsInstance(out, TrialTraceResult) - self.assertEqual(len(out), 11) - self.assertEqual(out[0].shape, (0, inp["a_final"].shape[1])) - self.assertEqual(out.d_f_denoised.shape, out.d_f.shape) - self.assertEqual(out.events.shape, out.d_f.shape) - self.assertEqual(out.f_residual.shape, out.d_f.shape) - self.assertEqual(out[4].shape, (0, 2)) # globalF (0, channels) - self.assertEqual(out[2].shape, (0,)) # frames - - def test_kept_trial_reads_and_computes(self): - """A kept trial reads (monkeypatched) then computes the traces.""" - for simple in (False, True): - with self.subTest(simple_trace_extraction=simple): - self._check_kept_trial(simple) - self._check_kept_trial(False, TransientOptions(max_iter=1500)) - - def _check_kept_trial(self, simple, transient_options=None): - """Verify IO wrapper forwards the selected extraction mode.""" - inp = _trial_inputs(num_channels=1) - g = inp["g"] - - def fake_load(*args, **kwargs): - """Return the synthetic loaded-trial arrays.""" - return ( - inp["data"], - inp["data2"], - inp["a_data"], - inp["frames"], - ) - - orig = tr._load_high_res_trial_data - tr._load_high_res_trial_data = fake_load - try: - out = tr.get_high_res_traces( - (0, True, inp["background_ds"]), - 0, - 100.0, - np.zeros((g["nsp"], 1), dtype=np.int32), - "unused", - {}, - g["smi"], - g["sh_inds"], - g["sh_vals"], - inp["a_final"].numpy(), # numpy -> exercises tensor conversion - inp["unique_motion_ds"], - inp["mot_keep_ds"], - 0.0, - g["psf2d"], - g["nsp"], - g["nz"], - g["npc"], - g["npr"], - 1, - inp["soma_sps"], - simple_trace_extraction=simple, - transient_options=transient_options, - ) - finally: - tr._load_high_res_trial_data = orig - - # matches a direct compute with the same inputs - expected = _compute( - inp, simple_trace_extraction=simple, - transient_options=transient_options, analyze_hz=100, - ) - for field in ("d_f", "f0_ls", "d_f_denoised", "events"): - np.testing.assert_allclose( - getattr(out, field), getattr(expected, field), - rtol=1e-6, equal_nan=True, - ) - self.assertEqual(out[0].shape, (inp["n_frames"], 3)) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_bandsilo_transient.py b/tests/test_bandsilo_transient.py deleted file mode 100644 index 4a23d9d..0000000 --- a/tests/test_bandsilo_transient.py +++ /dev/null @@ -1,169 +0,0 @@ -"""Fixed-profile temporal fitting: motion, gaps, boundaries and gradients.""" - -import unittest -from unittest.mock import patch - -import numpy as np -from scipy.optimize import OptimizeResult - -from giant_python.extraction.band import transient as tt - - -def synthetic_problem(): - """Two overlapping sources, alternating motion and an observation gap.""" - designs = { - 0: np.array([[1, .2], [.1, 1], [.5, .3]], dtype=np.float32), - 1: np.array([[.2, 1], [1, .1], [.4, .8]], dtype=np.float32), - } - events = np.zeros((65, 2)) - events[0] = [.4, .2] - events[15, 0], events[38, 1] = 2, 3 - expected = tt.TransientOperator(50, .15).forward(events) - motion = np.arange(65) % 2 - residual = np.stack([ - designs[int(m)] @ expected[t] for t, m in enumerate(motion) - ], axis=1).astype(np.float32) - valid = np.ones(65, dtype=bool) - valid[9:12] = False - residual[:, 9:12] = np.nan - return residual, designs, motion, valid, expected, events - - -class TestTransientSolver(unittest.TestCase): - """Numerical contracts independent of acquisition files and localization.""" - - def test_adjoint_and_gradient(self): - """Analytic derivatives agree with directional finite differences.""" - rng = np.random.default_rng(11) - s, z = rng.normal(size=(40, 2)), rng.normal(size=(40, 2)) - groups = {0: np.arange(0, 40, 2), 1: np.arange(1, 40, 2)} - normal = {} - for m, ix in groups.items(): - x = rng.normal(size=(3, 2)) - normal[m] = x.T @ x, rng.normal(size=(2, len(ix))) - for kernel in (None, [1, .8, .2]): - op = tt.TransientOperator(50, .15, kernel) - self.assertAlmostEqual( - np.sum(op.forward(s) * z), np.sum(s * op.adjoint(z)), places=10 - ) - _, grad = tt._temporal_quadratic(normal, groups, op.forward(s)) - direction = rng.normal(size=s.shape) - eps = 1e-5 - hi = tt._temporal_quadratic(normal, groups, op.forward(s + eps * direction))[0] - lo = tt._temporal_quadratic(normal, groups, op.forward(s - eps * direction))[0] - self.assertAlmostEqual( - (hi - lo) / (2 * eps), np.sum(direction * op.adjoint(grad)), places=5 - ) - - def test_recovery_masks_initial_state_and_independent_trials(self): - """Motion changes and missing frames never reset the decay state.""" - r, designs, motion, valid, expected, events = synthetic_problem() - options = tt.TransientOptions(beta=0, max_iter=1500, tolerance=1e-12) - result = tt.fit_transient_traces(r, designs, motion, valid, 50, options=options) - self.assertTrue(result.metadata["converged"]) - np.testing.assert_allclose(result.d_f[valid], expected[valid], atol=5e-4) - np.testing.assert_allclose(result.initial_state, events[0], atol=5e-4) - self.assertTrue(np.isnan(result.d_f[~valid]).all()) - self.assertTrue(np.isfinite(result.model_trace).all()) - self.assertTrue((result.events >= 0).all()) - self.assertTrue((result.events[0] == 0).all()) - self.assertEqual(np.argmax(result.events[:, 0]), 15) - self.assertEqual(np.argmax(result.events[:, 1]), 38) - reconstructed = result.events.copy() - reconstructed[0] = result.initial_state - np.testing.assert_array_equal( - result.model_trace, tt.TransientOperator(50, .15).forward(reconstructed) - ) - second = tt.fit_transient_traces( - np.zeros_like(r), designs, motion, np.ones_like(valid), 50, options=options - ) - np.testing.assert_array_equal(second.model_trace, 0) - # Neither caller masks nor input data are modified. - np.testing.assert_array_equal(valid, synthetic_problem()[3]) - np.testing.assert_array_equal(r, synthetic_problem()[0]) - - def test_debias_and_event_penalty(self): - """Optional support refitting restores amplitudes without new events.""" - r, designs, motion, valid, expected, events = synthetic_problem() - settings = dict(beta=.2, max_iter=2000, tolerance=1e-12) - selected = tt.fit_transient_traces( - r, designs, motion, valid, 50, options=tt.TransientOptions(**settings) - ) - result = tt.fit_transient_traces( - r, designs, motion, valid, 50, - options=tt.TransientOptions(**settings, debias=True, event_threshold=.01), - ) - self.assertTrue(result.metadata["converged"]) - self.assertTrue(result.metadata["debias"]["performed"]) - self.assertEqual(result.metadata["final_event_penalty"], 0) - np.testing.assert_allclose(result.d_f[valid], expected[valid], atol=5e-4) - np.testing.assert_allclose(result.events[1:], events[1:], atol=5e-4) - self.assertTrue((result.events[selected.events <= .01] == 0).all()) - - def test_nonconvergence_and_failed_debias(self): - """Both optimization stages expose failures, never silently succeed.""" - r, designs, motion, valid, *_ = synthetic_problem() - options = tt.TransientOptions(max_iter=1, debias=True) - with self.assertWarns(RuntimeWarning): - result = tt.fit_transient_traces(r, designs, motion, valid, 50, options=options) - self.assertFalse(result.metadata["converged"]) - self.assertFalse(result.metadata["debias"]["performed"]) - success = OptimizeResult(x=np.ones(130), fun=0., success=True, nit=2, message="ok") - failed = OptimizeResult(x=np.ones(130), fun=0., success=False, nit=1, message="limit") - with patch.object(tt, "minimize", side_effect=[success, failed]): - with self.assertWarns(RuntimeWarning): - result = tt.fit_transient_traces( - r, designs, motion, valid, 50, - options=tt.TransientOptions(debias=True, event_threshold=0), - ) - self.assertFalse(result.metadata["converged"]) - self.assertTrue(result.metadata["debias"]["performed"]) - - def test_invalid_settings_and_inputs(self): - """Reject malformed arrays, rates and optimization options.""" - for key, values in { - "tau_s": [0, -1, np.nan, True], - "beta_fraction": [-1, np.inf, True], - "beta": [-1, np.nan, True], - "max_iter": [0, 1.5, True], - "tolerance": [0, np.inf], - "debias": [1, "true"], - "event_threshold": [-1, np.nan], - }.items(): - for value in values: - with self.subTest(key=key, value=value), self.assertRaises(ValueError): - tt.TransientOptions(**{key: value}) - r, designs, motion, valid, *_ = synthetic_problem() - for rate in (None, 0, -1, np.nan, True): - with self.assertRaises(ValueError): - tt.fit_transient_traces(r, designs, motion, valid, rate) - for kernel in ([0, 1], [], [-1, 2], [1, np.nan]): - with self.assertRaises(ValueError): - tt.TransientOperator(50, .15, kernel) - for rr, xx, mm, vv in ( - (r.ravel(), designs, motion, valid), - (r, designs, motion[:-1], valid), - (r, {}, motion, valid), - (r, {0: np.ones(3)}, motion, valid), - (r, {0: np.full((3, 2), np.nan)}, motion, valid), - (r, {0: np.zeros((3, 2))}, motion, valid), - (r, designs, motion, np.zeros_like(valid)), - ): - with self.assertRaises(ValueError): - tt.fit_transient_traces(rr, xx, mm, vv, 50) - - def test_rank_deficiency_and_nonfinite_complete_frames(self): - """No inverse is required, and individual bad samples mask whole frames.""" - r, designs, motion, valid, *_ = synthetic_problem() - for x in designs.values(): - x[:, 1] = x[:, 0] - r[0, 1] = np.inf - motion[5] = -1 - result = tt.fit_transient_traces(r, designs, motion, valid, 50) - self.assertTrue(result.metadata["converged"]) - self.assertTrue(np.isnan(result.d_f[[1, 5, 9, 10, 11]]).all()) - self.assertTrue(np.isfinite(result.d_f[[0, 2, 3, 4]]).all()) - - -if __name__ == "__main__": - unittest.main() \ No newline at end of file diff --git a/tests/test_bandsilo_trial_data.py b/tests/test_bandsilo_trial_data.py deleted file mode 100644 index 0dd9ae9..0000000 --- a/tests/test_bandsilo_trial_data.py +++ /dev/null @@ -1,299 +0,0 @@ -"""Tests for canonical band trial reduction and shared numerical helpers.""" - -import os -import tempfile -import unittest - -import numpy as np - -from giant_python.extraction.band import trial_data as td -from giant_python.numerics import interpolation, morphology - - -class _MockDataFile: - """Minimal SLAP2 data-file stub for accumulate_superpixel_data. - - Models a single cycle with ``lines_per_cycle`` lines. Each line maps to a - superpixel id / fast-Z via ``line_super_pixel_ids`` and ``line_fastz`` and - returns a fixed per-channel value from ``get_line_data``. - """ - - def __init__(self, lines_per_cycle, num_cycles, line_specs, channels=2): - """Build a stub from per-line ``(sp_ids, fastz, values)`` specs.""" - self.header = {"linesPerCycle": lines_per_cycle} - self.numCycles = num_cycles - self._line_specs = line_specs - self._channels = channels - - class _Meta: - """Minimal metadata stub exposing ``linePeriod_s``.""" - - linePeriod_s = 1.0 - - self.metaData = _Meta() - self.lineDataNumElements = [len(spec[0]) for spec in line_specs] - self.lineSuperPixelIDs = [np.asarray(spec[0]) for spec in line_specs] - self.lineFastZIdxs = [np.asarray(spec[1]) for spec in line_specs] - - def getLineData(self, lines, cycles, channel): - """Return a per-line list of (n_positions, n_channels) arrays.""" - out = [] - for li in lines: - spec = self._line_specs[int(li) - 1] - values = np.asarray(spec[2], dtype=np.float32) - out.append(np.tile(values.reshape(-1, 1), (1, self._channels))) - return out - - -class TestNearestInterp(unittest.TestCase): - """nearest_interp mirrors MATLAB nearest interpolation.""" - - def test_single_point(self): - """A single sample returns yp unchanged.""" - yp = np.array([7.0]) - out = interpolation.nearest_interp( - np.array([0.0, 5.0]), np.array([2.0]), yp - ) - np.testing.assert_array_equal(out, yp) - - def test_midpoint_rounding(self): - """Query points snap to the nearest sample boundary.""" - xp = np.array([0.0, 10.0]) - yp = np.array([1.0, 2.0]) - out = interpolation.nearest_interp(np.array([4.0, 6.0]), xp, yp) - np.testing.assert_array_equal(out, np.array([1.0, 2.0])) - - -class TestFastDilation(unittest.TestCase): - """fast_dilation covers the 3x3 fast path and the cv2 fallback.""" - - def test_3x3_fast_path(self): - """A single point dilates to its 8-neighborhood.""" - mask = np.zeros((5, 5), dtype=bool) - mask[2, 2] = True - out = morphology.fast_dilation(mask, np.ones((3, 3), np.uint8)) - self.assertEqual(out[1:4, 1:4].sum(), 9) - self.assertEqual(out.sum(), 9) - - def test_leading_axes_fast_path(self): - """The fast path dilates only the trailing two axes.""" - mask = np.zeros((2, 5, 5), dtype=bool) - mask[0, 2, 2] = True - out = morphology.fast_dilation(mask, np.ones((3, 3), np.uint8)) - self.assertEqual(out[0].sum(), 9) - self.assertEqual(out[1].sum(), 0) - - def test_generic_cv2_path(self): - """A non-3x3 kernel routes through the cv2 fallback.""" - mask = np.zeros((5, 5), dtype=bool) - mask[2, 2] = True - out = morphology.fast_dilation(mask, np.ones((1, 3), np.uint8)) - # 1x3 horizontal kernel dilates left/right only. - self.assertTrue(out[2, 1] and out[2, 3] and out[2, 2]) - self.assertFalse(out[1, 2] or out[3, 2]) - - def test_default_kernel(self): - """A None kernel uses the 7x7 default (generic path).""" - mask = np.zeros((9, 9), dtype=bool) - mask[4, 4] = True - out = morphology.fast_dilation(mask) - self.assertEqual(out[1:8, 1:8].sum(), 49) - - -class TestComputeDsFrames(unittest.TestCase): - """compute_ds_frames builds the downsampled line grid.""" - - def test_grid(self): - """The grid is ceil of arange(first, last+1, dt).""" - out = td.compute_ds_frames(1, 5, 2.0) - np.testing.assert_array_equal(out, np.array([1.0, 3.0, 5.0])) - - -class TestAccumulate(unittest.TestCase): - """accumulate_superpixel_data scatters weighted line data.""" - - def _super_pixel_ids(self): - """Two superpixels: sp1 z1 -> 101, sp2 z1 -> 201.""" - return np.array([[101], [201]], dtype=np.int32) - - def _data_file(self, channels=2): - """One cycle, two lines each mapping to one superpixel.""" - line_specs = [ - ([1], [1], [10.0]), # line 1 -> sp id 1*100+1 = 101, value 10 - ([2], [1], [20.0]), # line 2 -> sp id 2*100+1 = 201, value 20 - ] - return _MockDataFile(2, 1, line_specs, channels=channels) - - def test_single_channel(self): - """Channel-1 read populates data/data_count; no second channel.""" - df = self._data_file() - ds_frames = np.array([1.0, 2.0]) - acc = td.accumulate_superpixel_data( - df, ds_frames, 1.0, self._super_pixel_ids(), 1, False - ) - self.assertEqual(acc["data"].shape, (2, 2)) - self.assertIsNone(acc["data2"]) - # sp1 gets its strongest weight at frame 0 (line 1), sp2 at frame 1. - self.assertGreater(acc["data"][0, 0], 0) - self.assertGreater(acc["data"][1, 1], 0) - # Normalized value recovers the line value. - norm = acc["data"][0, 0] / acc["data_count"][0, 0] - self.assertAlmostEqual(norm, 10.0, places=4) - - def test_two_channels(self): - """all_channels + num_channels>=2 fills the second-channel arrays.""" - df = self._data_file() - ds_frames = np.array([1.0, 2.0]) - acc = td.accumulate_superpixel_data( - df, ds_frames, 1.0, self._super_pixel_ids(), 2, True - ) - self.assertIsNotNone(acc["data2"]) - self.assertEqual(acc["data2"].shape, (2, 2)) - self.assertGreater(acc["data_count2"][0, 0], 0) - - def test_zero_element_line_skipped(self): - """A line with zero data elements is skipped.""" - line_specs = [ - ([], [], []), # line 1: empty -> skipped - ([2], [1], [20.0]), - ] - df = _MockDataFile(2, 1, line_specs, channels=1) - acc = td.accumulate_superpixel_data( - df, np.array([1.0, 2.0]), 1.0, self._super_pixel_ids(), 1, False - ) - # sp1 (id 101) never accumulates because its only line is empty. - self.assertEqual(acc["data"][0].sum(), 0) - - def test_line_with_no_matching_superpixels_skipped(self): - """A non-empty line matching none of the ids is skipped.""" - line_specs = [ - ([9], [9], [10.0]), # lookup 909 -> matches neither 101 nor 201 - ([2], [1], [20.0]), # sp2 (id 201) matches - ] - df = _MockDataFile(2, 1, line_specs, channels=1) - acc = td.accumulate_superpixel_data( - df, np.array([1.0, 2.0]), 1.0, self._super_pixel_ids(), 1, False - ) - # sp1 (101) never matches any line; sp2 (201) accumulates. - self.assertEqual(acc["data"][0].sum(), 0) - self.assertGreater(acc["data"][1].sum(), 0) - - -class TestReadBandTrialData(unittest.TestCase): - """read_band_trial_data orchestrates open+accumulate+alignment.""" - - def test_skipped_trial(self): - """keep_trial=False returns None without opening anything.""" - out = td.read_band_trial_data( - 0, False, 0, 100.0, np.zeros((1, 1)), "/data", {}, 1 - ) - self.assertIsNone(out) - - def test_full_read_with_monkeypatched_open(self): - """A kept trial reads, scales by 1/100, and loads alignment.""" - sp_ids = np.array([[101], [201]], dtype=np.int32) - df = _MockDataFile( - 2, 1, [([1], [1], [10.0]), ([2], [1], [20.0])], channels=1 - ) - trial_table = { - "filename": np.array([["t1.dat"]], dtype=object), - "first_line": np.array([[1]]), - "last_line": np.array([[2]]), - "fn_adata": np.array([["a.h5"]], dtype=object), - } - - orig_open = td.open_slap2_file - orig_align = td.load_alignment_data_h5 - td.open_slap2_file = lambda path: df - td.load_alignment_data_h5 = lambda path: { - "motionDSr": np.zeros(2), - "motionDSc": np.zeros(2), - "motionDSz": np.zeros(2), - } - try: - out = td.read_band_trial_data( - 0, - True, - 0, - 1.0, - sp_ids, - "/data", - trial_table, - 1, - all_channels=False, - ) - finally: - td.open_slap2_file = orig_open - td.load_alignment_data_h5 = orig_align - - self.assertIsNotNone(out) - self.assertEqual(out["data"].shape, (2, 2)) - self.assertIsNone(out["data2"]) - norm = out["data"][0, 0] / out["data_count"][0, 0] - self.assertAlmostEqual(norm, 10.0 / 100.0, places=6) - - -class TestAssembleAndCache(unittest.TestCase): - """assemble_lowres_data + save/load npz round trip.""" - - def _results(self, num_channels): - """Two synthetic trial results (one None) for assembly.""" - r0 = { - "data": np.ones((2, 3), dtype=np.float32), - "data_count": np.full((2, 3), 2.0, dtype=np.float32), - "alignment": { - "motionDSr": np.zeros(3), - "motionDSc": np.ones(3), - "motionDSz": np.full(3, 2.0), - }, - "ds_frames": np.arange(3), - "data2": np.full((2, 3), 5.0, dtype=np.float32), - "data_count2": np.full((2, 3), 2.0, dtype=np.float32), - } - r1 = { - "data": np.ones((2, 2), dtype=np.float32), - "data_count": np.full((2, 2), 2.0, dtype=np.float32), - "alignment": { - "motionDSr": np.zeros(2), - "motionDSc": np.ones(2), - "motionDSz": np.full(2, 2.0), - }, - "ds_frames": np.arange(2), - "data2": np.full((2, 2), 5.0, dtype=np.float32), - "data_count2": np.full((2, 2), 2.0, dtype=np.float32), - } - return [r0, None, r1] - - def test_assemble_single_channel(self): - """Single-channel assembly concatenates and builds trial IDs.""" - arrays = td.assemble_lowres_data(self._results(1), 1) - self.assertEqual(arrays["lowResData"].shape, (2, 5)) - self.assertNotIn("lowResData2", arrays) - # trial IDs: three 0s (r0) then two 2s (r1, original index 2). - np.testing.assert_array_equal( - arrays["lowResTrialID"], np.array([0, 0, 0, 2, 2]) - ) - - def test_assemble_two_channels(self): - """Two-channel assembly adds the second-channel arrays.""" - arrays = td.assemble_lowres_data(self._results(2), 2) - self.assertEqual(arrays["lowResData2"].shape, (2, 5)) - self.assertEqual(arrays["lowResDataCt2"].shape, (2, 5)) - - def test_save_load_round_trip(self): - """save_lowres_data + load_lowres_data preserve arrays.""" - arrays = td.assemble_lowres_data(self._results(2), 2) - with tempfile.TemporaryDirectory() as d: - path = os.path.join(d, "lowres_data_DMD1.npz") - td.save_lowres_data(path, arrays) - loaded = td.load_lowres_data(path, 2) - np.testing.assert_array_equal( - loaded["lowResData"], arrays["lowResData"] - ) - np.testing.assert_array_equal( - loaded["lowResData2"], arrays["lowResData2"] - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_cli.py b/tests/test_cli.py deleted file mode 100644 index b6635c3..0000000 --- a/tests/test_cli.py +++ /dev/null @@ -1,369 +0,0 @@ -"""Tests for the giant_python.cli argument parser and dispatch.""" - -import io -import subprocess -import sys -import unittest -from unittest import mock - -from giant_python import cli - - -class TestBuildParser(unittest.TestCase): - """build_parser exposes only annotation and extraction with valid modes.""" - - def setUp(self): - """Build a fresh parser for each test.""" - self.parser = cli.build_parser() - - def test_no_command(self): - """With no subcommand, command is None.""" - args = self.parser.parse_args([]) - self.assertIsNone(args.command) - - def test_removed_commands_are_absent(self): - """Removed stages have neither CLI help entries nor handlers.""" - help_text = self.parser.format_help() - for command in ("annotate", "extract"): - self.assertIn(command, help_text) - for command in ("organize", "register"): - with self.subTest(command=command): - self.assertNotIn(command, help_text) - self.assertFalse(hasattr(cli, "_cmd_" + command)) - with mock.patch( - "sys.stderr", new_callable=io.StringIO - ) as stderr: - with self.assertRaises(SystemExit) as error: - self.parser.parse_args([command]) - self.assertEqual(error.exception.code, 2) - self.assertIn("invalid choice", stderr.getvalue()) - self.assertIn(command, stderr.getvalue()) - - def test_operator_flag_is_rejected(self): - """Neither command accepts the removed operator option.""" - for command in ("annotate", "extract"): - with self.subTest(command=command): - with mock.patch("sys.stderr", new_callable=io.StringIO): - with self.assertRaises(SystemExit) as error: - self.parser.parse_args( - [command, "tt.h5", "--operator", "Ada"] - ) - self.assertEqual(error.exception.code, 2) - args = self.parser.parse_args([command, "tt.h5"]) - self.assertFalse(hasattr(args, "operator")) - - def test_annotate_defaults(self): - """annotate defaults to band scan, no draw, auto interactivity.""" - args = self.parser.parse_args(["annotate", "tt.h5"]) - self.assertEqual(args.command, "annotate") - self.assertEqual(args.trial_table, "tt.h5") - self.assertEqual(args.microscope, "slap2") - self.assertEqual(args.scan_mode, "band") - self.assertFalse(args.draw_user_rois) - self.assertIsNone(args.interactive) - self.assertFalse(args.verbose) - - def test_extract_flags(self): - """extract parses the draw and headless flags.""" - args = self.parser.parse_args( - ["extract", "tt.h5", "--draw-user-rois", "--headless"] - ) - self.assertTrue(args.draw_user_rois) - self.assertFalse(args.interactive) - - def test_interactive_flag(self): - """--interactive resolves to True.""" - args = self.parser.parse_args(["annotate", "tt.h5", "--interactive"]) - self.assertTrue(args.interactive) - - def test_supported_routing_choices(self): - """Both implemented commands accept explicit SLAP2 band routing.""" - for command in ("annotate", "extract"): - with self.subTest(command=command): - args = self.parser.parse_args( - [ - command, - "tt.h5", - "--microscope", - "slap2", - "--scan-mode", - "band", - ] - ) - self.assertEqual(args.microscope, "slap2") - self.assertEqual(args.scan_mode, "band") - - -class TestOptionsFromArgs(unittest.TestCase): - """CLI separates science, execution and annotation configuration.""" - - def test_defaults_have_no_trace_mode_fields(self): - """Both commands build options without removed mode fields.""" - parser = cli.build_parser() - for command in ("annotate", "extract"): - with self.subTest(command=command): - args = parser.parse_args([command, "tt.h5"]) - original = vars(args).copy() - params, _, _ = cli._options_from_args(args) - for name in ("simple_trace_extraction", "convolution_trace_extraction"): - self.assertFalse(hasattr(args, name)) - self.assertFalse(hasattr(params, name)) - self.assertEqual(vars(args), original) - - def test_trace_tuning_flags(self): - """Trace controls and the shared decay setting reach science options.""" - parser = cli.build_parser() - args = parser.parse_args([ - "extract", "tt.h5", - "--decay-tau-s", "0.25", "--trace-beta-fraction", "0.02", - "--trace-max-iter", "900", "--trace-tolerance", "1e-9", - "--trace-debias", "--trace-event-threshold", "0.001", - ]) - params, _, _ = cli._options_from_args(args) - self.assertEqual(params.decay_tau_s, .25) - self.assertEqual(params.trace_beta_fraction, .02) - self.assertEqual(params.trace_max_iter, 900) - self.assertEqual(params.trace_tolerance, 1e-9) - self.assertTrue(params.trace_debias) - self.assertEqual(params.trace_event_threshold, .001) - - def test_maps_fields(self): - """Run and annotation flags do not change scientific defaults.""" - args = cli.build_parser().parse_args( - [ - "extract", - "tt.h5", - "--scan-mode", - "band", - "--draw-user-rois", - "--interactive", - "--verbose", - ] - ) - params, execution, annotations = cli._options_from_args(args) - self.assertEqual(params.peakth, 7.0) - self.assertEqual(params.z_tol, 1.5) - self.assertFalse(hasattr(params, "scan_mode")) - self.assertTrue(annotations.enabled) - self.assertTrue(annotations.interactive) - self.assertFalse(hasattr(annotations, "operator")) - self.assertTrue(execution.verbose) - self.assertEqual(execution.max_workers, 6) - self.assertIsNone(execution.max_trials) - - def test_explicit_annotation_enables_rois(self): - """The annotate command itself enables ROI handling.""" - args = cli.build_parser().parse_args(["annotate", "tt.h5"]) - _, _, annotations = cli._options_from_args(args) - self.assertTrue(annotations.enabled) - - def test_run_limits(self): - """Execution limits map to validated options.""" - args = cli.build_parser().parse_args( - ["extract", "tt.h5", "--max-workers", "2", "--max-trials", "3"] - ) - _, execution, _ = cli._options_from_args(args) - self.assertEqual(execution.max_workers, 2) - self.assertEqual(execution.max_trials, 3) - - -class TestMain(unittest.TestCase): - """main dispatches to handlers and handles the no-command case.""" - - def test_help_does_not_import_extraction_or_gui_backends(self): - """Fresh-process CLI help exits successfully without heavy imports.""" - code = """ -import sys -from giant_python.cli import main -try: - main(sys.argv[1:]) -except SystemExit as error: - assert error.code == 0, error.code -else: - raise AssertionError('help did not exit') -blocked = ('torch', 'cv2', 'tkinter', 'slap2_utils', - 'giant_python.extraction.band.workflow', - 'giant_python.gui') -loaded = [name for name in sys.modules - if any(name == prefix or name.startswith(prefix + '.') - for prefix in blocked)] -assert not loaded, loaded -""" - for args in (["-h"], ["extract", "-h"], ["annotate", "-h"]): - with self.subTest(args=args): - result = subprocess.run( - [sys.executable, "-c", code, *args], - capture_output=True, - text=True, - timeout=30, - ) - self.assertEqual(result.returncode, 0, result.stderr) - self.assertIn("usage:", result.stdout) - - def test_no_command_prints_help(self): - """No subcommand prints help and returns 2.""" - parser = cli.build_parser() - with ( - mock.patch.object(cli, "build_parser", return_value=parser), - mock.patch.object(parser, "print_help") as help_, - ): - code = cli.main([]) - help_.assert_called_once() - self.assertEqual(code, 2) - - def test_dispatches_annotate(self): - """main routes the annotate command to its handler.""" - with mock.patch.object( - cli, "_cmd_annotate", return_value=0 - ) as handler: - code = cli.main(["annotate", "tt.h5"]) - handler.assert_called_once() - self.assertEqual(code, 0) - - -class TestCommandServices(unittest.TestCase): - """Handlers delegate to public services and report invalid requests.""" - - def test_extract_service(self): - """The extraction handler passes a filename, not a fake table.""" - with mock.patch( - "giant_python.pipeline.extract.extract_sources" - ) as extract: - code = cli.main( - [ - "extract", - "curated_v2.h5", - "--max-trials", - "4", - "--z-tol", - "2.5", - "--headless", - "--draw-user-rois", - ] - ) - self.assertEqual(code, 0) - self.assertEqual(extract.call_args.args[0], "curated_v2.h5") - self.assertEqual(extract.call_args.args[1].z_tol, 2.5) - for name in ("simple_trace_extraction", "convolution_trace_extraction"): - self.assertFalse(hasattr(extract.call_args.args[1], name)) - self.assertEqual(extract.call_args.kwargs["scan_mode"], "band") - self.assertEqual(extract.call_args.kwargs["execution"].max_trials, 4) - self.assertTrue(extract.call_args.kwargs["annotations"].enabled) - self.assertFalse(extract.call_args.kwargs["annotations"].interactive) - - def test_annotation_service(self): - """Annotation routes through the same public function as Pipeline.""" - with mock.patch( - "giant_python.pipeline.annotate.annotate_rois" - ) as annotate: - code = cli.main(["annotate", "curated_v2.h5", "--headless"]) - self.assertEqual(code, 0) - self.assertEqual(annotate.call_args.args[0], "curated_v2.h5") - self.assertTrue(annotate.call_args.kwargs["annotations"].enabled) - self.assertFalse(annotate.call_args.kwargs["annotations"].interactive) - - def test_removed_commands_fail_before_dispatch(self): - """Old commands cannot fall through to either implemented handler.""" - with ( - mock.patch.object(cli, "_cmd_annotate") as annotate, - mock.patch.object(cli, "_cmd_extract") as extract, - ): - for argv in ( - ["organize", "/data", "/results"], - ["register", "curated_v2.h5"], - ): - with self.subTest(argv=argv): - with mock.patch( - "sys.stderr", new_callable=io.StringIO - ) as stderr: - with self.assertRaises(SystemExit) as error: - cli.main(argv) - self.assertEqual(error.exception.code, 2) - self.assertIn("invalid choice", stderr.getvalue()) - self.assertIn(argv[0], stderr.getvalue()) - annotate.assert_not_called() - extract.assert_not_called() - - def test_removed_trace_flags_fail_before_dispatch(self): - """Each former flag is unknown, not a compatibility or mode switch.""" - flags = ("--simple-trace-extraction", "--convolution-trace-extraction") - with ( - mock.patch.object(cli, "_cmd_annotate") as annotate, - mock.patch.object(cli, "_cmd_extract") as extract, - ): - for command in ("annotate", "extract"): - for supplied in ([flags[0]], [flags[1]], list(flags)): - with self.subTest(command=command, flags=supplied): - with mock.patch("sys.stderr", new_callable=io.StringIO) as stderr: - with self.assertRaises(SystemExit) as error: - cli.main([command, "tt.h5", *supplied]) - self.assertEqual(error.exception.code, 2) - self.assertIn("unrecognized arguments", stderr.getvalue()) - for flag in supplied: - self.assertIn(flag, stderr.getvalue()) - with mock.patch("sys.stdout", new_callable=io.StringIO) as stdout: - with self.assertRaises(SystemExit) as error: - cli.main([command, "--help"]) - self.assertEqual(error.exception.code, 0) - for flag in flags: - self.assertNotIn(flag, stdout.getvalue()) - annotate.assert_not_called() - extract.assert_not_called() - - def test_unsupported_routing_reports_parser_error(self): - """Unsupported routing is rejected by argparse before dispatch.""" - with ( - mock.patch.object(cli, "_cmd_annotate") as annotate, - mock.patch.object(cli, "_cmd_extract") as extract, - ): - for command in ("extract", "annotate"): - for flag, mode in ( - ("--scan-mode", "standard"), - ("--scan-mode", "unknown"), - ("--microscope", "bergamo"), - ("--microscope", "unknown"), - ): - with self.subTest(command=command, flag=flag, mode=mode): - with mock.patch( - "sys.stderr", new_callable=io.StringIO - ) as stderr: - with self.assertRaises(SystemExit) as error: - cli.main([command, "tt.h5", flag, mode]) - self.assertEqual(error.exception.code, 2) - self.assertIn("invalid choice", stderr.getvalue()) - self.assertIn(flag, stderr.getvalue()) - self.assertIn(mode, stderr.getvalue()) - annotate.assert_not_called() - extract.assert_not_called() - - def test_invalid_execution_limit_is_reported(self): - """Bad run options fail before the service can load any data.""" - with mock.patch("sys.stderr", new_callable=io.StringIO) as stderr: - with self.assertRaises(SystemExit) as error: - cli.main(["extract", "tt.h5", "--max-workers", "0"]) - self.assertEqual(error.exception.code, 2) - self.assertIn("max_workers", stderr.getvalue()) - - def test_service_value_error_is_actionable(self): - """ValueError from either public service becomes a CLI usage error.""" - services = { - "annotate": "giant_python.pipeline.annotate.annotate_rois", - "extract": "giant_python.pipeline.extract.extract_sources", - } - for command, target in services.items(): - with self.subTest(command=command): - with ( - mock.patch( - target, side_effect=ValueError("Invalid trial table") - ) as service, - mock.patch("sys.stderr", new_callable=io.StringIO) as err, - ): - with self.assertRaises(SystemExit) as error: - cli.main([command, "tt.h5"]) - service.assert_called_once() - self.assertEqual(error.exception.code, 2) - self.assertIn("Invalid trial table", err.getvalue()) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_configuration.py b/tests/test_configuration.py deleted file mode 100644 index e4ec884..0000000 --- a/tests/test_configuration.py +++ /dev/null @@ -1,266 +0,0 @@ -"""Contracts for scientific configuration and independent run policies.""" - -import math -import unittest -from copy import deepcopy -from dataclasses import asdict, fields - -from giant_python.models.params import ( - AnnotationOptions, - BandSiloParams, - ExecutionOptions, - resolve_band_options, -) - - -class TestBandConfiguration(unittest.TestCase): - """Resolution preserves numerical defaults and never mutates inputs.""" - - def test_defaults_match_existing_science(self): - """The split is architectural, not a scientific retuning.""" - science, execution, annotations = resolve_band_options() - self.assertEqual(science.peakth, 7.0) - self.assertEqual(science.denoise_window_s, 1.0) - self.assertEqual(science.vif, 1.38) - self.assertEqual(science.z_tol, 1.5) - for name in ("simple_trace_extraction", "convolution_trace_extraction"): - with self.subTest(name=name): - self.assertFalse(hasattr(science, name)) - self.assertNotIn(name, {field.name for field in fields(BandSiloParams)}) - self.assertNotIn(name, asdict(science)) - self.assertEqual(science.sparse_fac, math.exp(-3.0)) - self.assertEqual(execution, ExecutionOptions(6, False, None)) - self.assertEqual(annotations, AnnotationOptions(False, None)) - self.assertFalse(hasattr(annotations, "operator")) - self.assertFalse(hasattr(science, "max_workers")) - self.assertFalse(hasattr(science, "scan_mode")) - self.assertFalse(hasattr(science, "interactive")) - - def test_science_rejects_mixed_and_routing_fields(self): - """Science dictionaries never dispatch other scopes or old names.""" - for name, value in ( - ("microscope", "slap2"), - ("scan_mode", "band"), - ("max_workers", 2), - ("max_trials", 3), - ("verbose", True), - ("operator", "Ada"), - ("enabled", True), - ("interactive", False), - ("draw_user_rois", True), - ("sigma_px", 1.5), - ("lambda_", 0.1), - ("phi", None), - ("tau_s", None), - ("photon_scale", None), - ): - with self.subTest(name=name): - with self.assertRaisesRegex(TypeError, name): - resolve_band_options({"peakth": 7.0, name: value}) - - def test_every_result_is_fresh(self): - """Typed inputs and each default result are independent objects.""" - original = (BandSiloParams(), ExecutionOptions(), AnnotationOptions()) - resolved = resolve_band_options(*original) - for actual, expected in zip(resolved, original): - self.assertEqual(actual, expected) - self.assertIsNot(actual, expected) - self.assertIsNot(resolve_band_options()[0], resolved[0]) - - def test_dictionary_inputs_and_overrides_are_not_modified(self): - """Each dictionary affects only its scope and is copied.""" - params = {"analyze_hz": 80.0, "z_tol": 2.5, "trace_debias": True} - execution = {"max_workers": 2, "verbose": True} - annotations = { - "enabled": True, - "interactive": False, - } - original = deepcopy((params, execution, annotations)) - science, run, roi = resolve_band_options( - params, execution, annotations - ) - self.assertEqual(science.analyze_hz, 80.0) - self.assertEqual(science.z_tol, 2.5) - self.assertTrue(science.trace_debias) - self.assertEqual(run, ExecutionOptions(2, True, None)) - self.assertEqual(roi, AnnotationOptions(True, False)) - self.assertEqual((params, execution, annotations), original) - - -class TestConfigurationValidation(unittest.TestCase): - """Reject unsupported settings before they reach numerical kernels.""" - - def test_trace_tuning_options_use_shared_decay(self): - """Trace fitting uses the existing shared decay configuration.""" - params, _, _ = resolve_band_options({ - "decay_tau_s": .3, - "trace_beta_fraction": 0, "trace_max_iter": 1500, - "trace_debias": True, - }) - self.assertEqual(params.decay_tau_s, .3) - self.assertFalse(hasattr(params, "trace_tau_s")) - self.assertEqual(params.trace_beta_fraction, 0) - self.assertTrue(params.trace_debias) - self.assertEqual(params.trace_max_iter, 1500) - for key, values in { - "trace_debias": [1, "true"], - "decay_tau_s": [0, -1, math.nan, True], - "trace_tolerance": [0, math.inf], - "trace_max_iter": [0, 1.5, True], - "trace_beta_fraction": [-1, math.nan, True], - "trace_event_threshold": [-1, math.inf], - "random_seed": [-1, 1.5, True, None], - }.items(): - for value in values: - with self.subTest(key=key, value=value), self.assertRaisesRegex(ValueError, key): - BandSiloParams(**{key: value}) - - def test_removed_trace_flags_reject_all_boolean_combinations(self): - """Supplying both removed keys fails instead of normalizing them.""" - for simple in (False, True): - for convolution in (False, True): - with self.subTest(simple=simple, convolution=convolution): - supplied: dict[str, bool] = { - "simple_trace_extraction": simple, - "convolution_trace_extraction": convolution, - } - original = supplied.copy() - with self.assertRaisesRegex(TypeError, "trace_extraction"): - BandSiloParams(**supplied) - with self.assertRaisesRegex(TypeError, "trace_extraction"): - resolve_band_options(supplied) - self.assertEqual(supplied, original) - - def test_removed_trace_flags_reject_model_and_dictionary_inputs(self): - """Each removed key is unknown regardless of its supplied value.""" - for name in ("simple_trace_extraction", "convolution_trace_extraction"): - for value in (False, True, 0, 1, "true", "false", None, [], 0.0): - with self.subTest(name=name, value=value): - supplied = {"peakth": 8.0, name: value} - original = deepcopy(supplied) - with self.assertRaisesRegex(TypeError, name): - BandSiloParams(**supplied) - with self.assertRaisesRegex(TypeError, name): - resolve_band_options(supplied) - self.assertEqual(supplied, original) - - def test_positive_scientific_values(self): - """Rates, windows and positive scale values must be finite.""" - for name in ( - "analyze_hz", - "decay_tau_s", - "baseline_window_s", - "denoise_window_s", - "vif", - "sparse_fac", - "peakth", - "z_tol", - ): - for value in (0, -1, math.nan, math.inf, True, "1", None): - with self.subTest(name=name, value=value): - with self.assertRaisesRegex(ValueError, name): - BandSiloParams(**{name: value}) - - def test_positive_counts(self): - """Spatial counts require positive integers, not floats or bools.""" - for name in ("d_xy", "peak_buffer", "psf_dilation"): - for value in (0, -1, 1.5, True): - with self.subTest(name=name, value=value): - with self.assertRaisesRegex(ValueError, name): - BandSiloParams(**{name: value}) - - def test_background_interpolation_is_not_configurable(self): - """Neither typed nor dictionary options accept interpolation modes.""" - for mode in ("linear", "cubic", "nearest"): - with self.subTest(mode=mode): - options = {"background_interpolation": mode} - with self.assertRaisesRegex( - TypeError, "background_interpolation" - ): - BandSiloParams(**options) - with self.assertRaisesRegex( - TypeError, "background_interpolation" - ): - resolve_band_options(options) - - def test_channel_parameters_are_not_configurable(self): - """Removed channel options fail for both typed and dictionary inputs.""" - for name, value in ( - ("num_channels", None), - ("num_channels", 2), - ("activity_channel", 0), - ("activity_channel", 1), - ): - with self.subTest(name=name, value=value): - options = {name: value} - with self.assertRaisesRegex(TypeError, name): - BandSiloParams(**options) - with self.assertRaisesRegex(TypeError, name): - resolve_band_options(options) - - def test_operator_is_not_configurable(self): - """Removed operator metadata is rejected rather than ignored.""" - for value in ("Ada", "SLAP2 User", None): - with self.subTest(value=value): - options = {"operator": value} - with self.assertRaisesRegex(TypeError, "operator"): - AnnotationOptions(**options) - with self.assertRaisesRegex(TypeError, "operator"): - resolve_band_options(annotations=options) - - def test_execution_and_annotation_policy(self): - """Execution counts and tri-state GUI settings are validated.""" - for name in ("max_workers", "max_trials"): - for value in (0, -1, 1.5, True): - with self.subTest(name=name, value=value): - with self.assertRaisesRegex(ValueError, name): - ExecutionOptions(**{name: value}) - with self.assertRaisesRegex(ValueError, "verbose"): - ExecutionOptions(verbose=1) - for name, value in ( - ("enabled", 1), - ("interactive", "false"), - ): - with self.subTest(name=name): - with self.assertRaisesRegex(ValueError, name): - AnnotationOptions(**{name: value}) - - def test_resolver_revalidates_mutated_carriers(self): - """Dataclass mutability cannot bypass validation at resolution.""" - science = BandSiloParams() - science.d_xy = 0 - with self.assertRaisesRegex(ValueError, "d_xy"): - resolve_band_options(science) - execution = ExecutionOptions() - execution.max_workers = 0 - with self.assertRaisesRegex(ValueError, "max_workers"): - resolve_band_options(execution=execution) - annotations = AnnotationOptions() - annotations.interactive = "false" - with self.assertRaisesRegex(ValueError, "interactive"): - resolve_band_options(annotations=annotations) - - def test_bad_input_and_unknown_keys(self): - """Typos and wrong container types do not silently use defaults.""" - for params in (42, {"analyze_hzz": 80}): - with self.assertRaises(TypeError): - resolve_band_options(params) - for keyword, value in ( - ("execution", False), - ("annotations", "headless"), - ("execution", {"max_worker": 2}), - ("annotations", {"draw_user_rois": True}), - ("execution", {"operator": "Ada"}), - ("execution", {"peakth": 7}), - ("annotations", {"max_workers": 2}), - ("annotations", {"scan_mode": "band"}), - ): - with self.subTest(keyword=keyword, value=value): - with self.assertRaises(TypeError): - resolve_band_options(**{keyword: value}) - with self.assertRaises(TypeError): - resolve_band_options({"enabled": True, "draw_user_rois": False}) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_example.py b/tests/test_example.py deleted file mode 100644 index e2f68ae..0000000 --- a/tests/test_example.py +++ /dev/null @@ -1,17 +0,0 @@ -"""Tests for the giant_python package.""" - -import unittest - - -class PackageTest(unittest.TestCase): - """Tests for the giant_python package.""" - - def test_import(self): - """Test that the giant_python package can be imported.""" - import giant_python - - self.assertIsNotNone(giant_python.__version__) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_experiment_summary_codec.py b/tests/test_experiment_summary_codec.py deleted file mode 100644 index 319d770..0000000 --- a/tests/test_experiment_summary_codec.py +++ /dev/null @@ -1,405 +0,0 @@ -"""Focused full-schema summary codec and legacy compatibility coverage.""" - -import unittest -from pathlib import Path -from tempfile import TemporaryDirectory - -import h5py -import numpy as np - -from giant_python.io.experiment_summary import read_summary, write_summary -from giant_python.models.experiment_summary import ( - FRAME_INFO_KEYS, - ExperimentSummary, - FrameInfo, - PathSummary, - Source, - UserRoi, - Visualizations, -) - - -def _path(n_sources=2, n_rois=2, annotation_enabled=True, frames=7): - """Use distinct values and non-square geometry to expose axis changes.""" - trace = np.arange(2 * frames, dtype=np.float32).reshape(2, frames) - profile = np.arange(60, dtype=np.float32).reshape(3, 4, 5) - return PathSummary( - sources=[ - Source( - profile=profile + index, - coords=np.array([index, index + 1, index + 2], np.int32), - df_ls=trace + index, - df_denoised=trace + 100 + index, - events=trace + 200 + index, - f0=trace + 300 + index, - snr=np.float32(index + 0.5), - ) - for index in range(n_sources) - ], - user_rois=[ - UserRoi( - label=f"ROI μ {index}", - mask=(profile % (index + 2)) == 0, - f=trace + 400 + index, - fsvd=trace + 500 + index, - ) - for index in range(n_rois) - ], - visualizations=Visualizations( - mean_im=np.stack([profile, profile + 10]), - act_im=profile + 20, - act_im_peaks=np.zeros((n_sources, 3), np.int32), - ), - frame_info=FrameInfo(**{ - key: (np.arange(frames, dtype=np.float32) - index).reshape(-1, 1) - for index, key in enumerate(FRAME_INFO_KEYS) - }), - global_f=trace + 600, - z_depths=np.array([[2], [4], [8]], dtype=np.int16), - annotation_enabled=annotation_enabled, - ) - - -def _snapshot(destination): - """Capture datasets, groups and attributes without relying on HDF5 bytes.""" - snapshot = {} - with h5py.File(destination, "r") as file: - def collect(name, item): - """Keep values, dtype, shape and all metadata for comparison.""" - snapshot[(name, "kind")] = type(item).__name__ - for key, value in item.attrs.items(): - snapshot[(name, "attribute", key)] = value - if isinstance(item, h5py.Dataset): - snapshot[(name, "value")] = item[()] - snapshot[(name, "dtype")] = item.dtype - snapshot[(name, "shape")] = item.shape - - collect("", file) - file.visititems(collect) - return snapshot - - -class TestExperimentSummaryCodec(unittest.TestCase): - """Optional data stay genuine, missing data stay explicitly unavailable.""" - - def assert_full_schema(self, file, sources, rois, frames=7): - """Verify the full row-major schema, including zero-length axes.""" - group = file["Path1"] - self.assertEqual(set(group), { - "Z_depths", "sources", "frame_info", "global", - "visualizations", "user_rois", - }) - self.assertEqual(group["Z_depths"].shape, (3, 1)) - self.assertEqual(set(group["sources"]), {"spatial", "temporal"}) - spatial = group["sources/spatial"] - self.assertEqual(set(spatial), {"profiles", "coords"}) - self.assertEqual(spatial["profiles"].shape, (sources, 3, 4, 5)) - self.assertEqual(spatial["coords"].shape, (sources, 3)) - temporal = group["sources/temporal"] - self.assertEqual(set(temporal), { - "dF_ls", "dF_denoised", "events", "F0", "SNR", - }) - for key in ("dF_ls", "dF_denoised", "events", "F0"): - self.assertEqual(temporal[key].shape, (sources, 2, frames)) - self.assertEqual(temporal["SNR"].shape, (sources, 1)) - roi_group = group["user_rois"] - self.assertEqual(set(roi_group), {"mask", "labels", "F", "Fsvd"}) - self.assertEqual(roi_group["mask"].shape, (rois, 3, 4, 5)) - self.assertEqual(roi_group["labels"].shape, (rois, 1)) - for key in ("F", "Fsvd"): - self.assertEqual(roi_group[key].shape, (rois, 2, frames)) - self.assertEqual(set(group["frame_info"]), set(FRAME_INFO_KEYS)) - self.assertEqual(set(group["global"]), {"F"}) - self.assertEqual(group["global/F"].shape, (2, frames)) - self.assertEqual(set(group["visualizations"]), { - "mean_im", "act_im", "act_im_peaks", - }) - - def test_supplied_arrays_roundtrip_without_scientific_changes(self): - """All new and existing arrays preserve values, dtype and orientation.""" - original = _path() - original.sources[0].df_denoised[0, 1] = np.nan - original.sources[1].events = original.sources[1].events.astype(np.int16) - original.user_rois[0].fsvd = original.user_rois[0].fsvd.astype(np.float64) - with TemporaryDirectory() as directory: - destination = Path(directory) / "summary.h5" - rewritten = Path(directory) / "rewritten.h5" - write_summary(ExperimentSummary( - paths=[original], params={"operator": "μ", "channels": 2} - ), destination) - with h5py.File(destination, "r") as file: - self.assert_full_schema(file, 2, 2) - self.assertEqual(file["row_major"][()], 1) - self.assertTrue( - file["Path1/user_rois"].attrs["annotation_enabled"] - ) - for dataset, field in ( - ("dF_denoised", "df_denoised"), ("events", "events") - ): - np.testing.assert_array_equal( - file[f"Path1/sources/temporal/{dataset}"][()], - np.stack([getattr(s, field) for s in original.sources]), - ) - np.testing.assert_array_equal( - file["Path1/user_rois/Fsvd"][()], - np.stack([roi.fsvd for roi in original.user_rois]), - ) - summary = read_summary(destination) - loaded = summary.paths[0] - self.assertTrue(loaded.annotation_enabled) - self.assertEqual(summary.params["operator"], "μ") - for before, after in zip(original.sources, loaded.sources): - for field in ( - "profile", "coords", "df_ls", "df_denoised", - "events", "f0", "snr", - ): - np.testing.assert_array_equal( - getattr(after, field), getattr(before, field) - ) - for before, after in zip(original.user_rois, loaded.user_rois): - self.assertEqual(after.label, before.label) - for field in ("mask", "f", "fsvd"): - np.testing.assert_array_equal( - getattr(after, field), getattr(before, field) - ) - for key in FRAME_INFO_KEYS: - np.testing.assert_array_equal( - getattr(loaded.frame_info, key), - getattr(original.frame_info, key), - ) - for field in ("mean_im", "act_im", "act_im_peaks"): - np.testing.assert_array_equal( - getattr(loaded.visualizations, field), - getattr(original.visualizations, field), - ) - np.testing.assert_array_equal(loaded.global_f, original.global_f) - np.testing.assert_array_equal(loaded.z_depths, original.z_depths) - write_summary(summary, rewritten) - before, after = _snapshot(destination), _snapshot(rewritten) - self.assertEqual(before.keys(), after.keys()) - for key in before: - np.testing.assert_equal(after[key], before[key]) - - def test_missing_optional_arrays_are_nan_not_copies_or_zeros(self): - """Mixed missing/present traces are handled independently per entry.""" - for integer_reference in (False, True): - with self.subTest(integer_reference=integer_reference): - original = _path() - original.sources[0].df_denoised = None - original.sources[1].events = None - original.sources[0].snr = None - original.user_rois[0].fsvd = None - if integer_reference: - for source in original.sources: - source.df_ls = source.df_ls.astype(np.int16) - for roi in original.user_rois: - roi.f = roi.f.astype(np.int16) - with TemporaryDirectory() as directory: - destination = Path(directory) / "summary.h5" - write_summary(ExperimentSummary(paths=[original]), destination) - loaded = read_summary(destination).paths[0] - for missing, reference in ( - (loaded.sources[0].df_denoised, loaded.sources[0].df_ls), - (loaded.sources[1].events, loaded.sources[1].df_ls), - (loaded.user_rois[0].fsvd, loaded.user_rois[0].f), - ): - self.assertEqual(missing.shape, reference.shape) - self.assertTrue(np.isnan(missing).all()) - self.assertTrue(np.isnan(loaded.sources[0].snr)) - np.testing.assert_array_equal( - loaded.sources[1].df_denoised, - original.sources[1].df_denoised, - ) - np.testing.assert_array_equal( - loaded.sources[0].events, original.sources[0].events - ) - np.testing.assert_array_equal( - loaded.user_rois[1].fsvd, original.user_rois[1].fsvd - ) - self.assertIsNone(original.sources[0].df_denoised) - self.assertIsNone(original.sources[1].events) - self.assertIsNone(original.user_rois[0].fsvd) - - def test_empty_sources_and_rois_always_have_full_rank(self): - """Geometry inference and legacy templates work for either ROI state.""" - for enabled in (False, True): - for templates in (False, True): - for frames in (0, 7): - with self.subTest( - enabled=enabled, templates=templates, frames=frames - ), TemporaryDirectory() as directory: - original = _path(0, 0, enabled, frames) - if templates: - original._empty_source_arrays = { - "profiles": np.empty((0, 3, 4, 5), np.float64), - "coords": np.empty((0, 3), np.int16), - "dF_ls": np.empty((0, 2, frames), np.float32), - "F0": np.empty((0, 2, frames), np.float32), - "SNR": np.empty((0, 1), np.float32), - } - original._empty_roi_arrays = { - "mask": np.empty((0, 3, 4, 5), bool), - "F": np.empty((0, 2, frames), np.float64), - } - destination = Path(directory) / "summary.h5" - rewritten = Path(directory) / "rewritten.h5" - write_summary( - ExperimentSummary(paths=[original]), destination - ) - loaded = read_summary(destination) - self.assertEqual(loaded.paths[0].sources, []) - self.assertEqual(loaded.paths[0].user_rois, []) - self.assertEqual( - loaded.paths[0].annotation_enabled, enabled - ) - write_summary(loaded, rewritten) - for filename in (destination, rewritten): - with h5py.File(filename, "r") as file: - self.assert_full_schema(file, 0, 0, frames) - self.assertEqual( - bool(file["Path1/user_rois"].attrs[ - "annotation_enabled" - ]), enabled, - ) - if templates: - self.assertEqual( - file["Path1/sources/spatial/profiles"].dtype, - np.dtype(np.float64), - ) - self.assertEqual( - file["Path1/user_rois/F"].dtype, - np.dtype(np.float64), - ) - self.assertNotIn( - "dF_denoised", original._empty_source_arrays - ) - self.assertNotIn("Fsvd", original._empty_roi_arrays) - - def test_legacy_missing_datasets_and_annotation_metadata(self): - """Old absent/present ROI groups preserve their original semantics.""" - for sources in (0, 2): - for state in ("disabled", "empty", "populated"): - with self.subTest(sources=sources, state=state): - rois = 2 if state == "populated" else 0 - original = _path(sources, rois, state != "disabled") - with TemporaryDirectory() as directory: - destination = Path(directory) / "legacy.h5" - rewritten = Path(directory) / "upgraded.h5" - write_summary( - ExperimentSummary(paths=[original]), destination - ) - with h5py.File(destination, "a") as file: - for key in ("dF_denoised", "events"): - del file[f"Path1/sources/temporal/{key}"] - if state == "disabled": - del file["Path1/user_rois"] - else: - del file["Path1/user_rois/Fsvd"] - del file["Path1/user_rois"].attrs[ - "annotation_enabled" - ] - summary = read_summary(destination) - loaded = summary.paths[0] - self.assertEqual( - loaded.annotation_enabled, state != "disabled" - ) - for source in loaded.sources: - self.assertIsNone(source.df_denoised) - self.assertIsNone(source.events) - for roi in loaded.user_rois: - self.assertIsNone(roi.fsvd) - write_summary(summary, rewritten) - with h5py.File(rewritten, "r") as file: - self.assert_full_schema(file, sources, rois) - for key in ("dF_denoised", "events"): - self.assertTrue(np.isnan( - file[f"Path1/sources/temporal/{key}"][()] - ).all()) - self.assertTrue(np.isnan( - file["Path1/user_rois/Fsvd"][()] - ).all()) - self.assertEqual( - read_summary(rewritten).paths[0].annotation_enabled, - state != "disabled", - ) - - def test_optional_shape_errors_do_not_replace_existing_path(self): - """No broadcasting, transposing or reshaping supplied optional data.""" - for owner, field in ( - ("Source", "df_denoised"), ("Source", "events"), - ("UserRoi", "fsvd"), - ): - for shape in ((7,), (1, 7), (2, 6), (7, 2), (2, 7, 1)): - with self.subTest(owner=owner, field=field, shape=shape): - original = _path() - unrelated = _path(0, 0, False) - unrelated.path_index = 4 - with TemporaryDirectory() as directory: - destination = Path(directory) / "summary.h5" - write_summary(ExperimentSummary( - paths=[original, unrelated], - params={"operator": "original"}, - ), destination) - with h5py.File(destination, "a") as file: - file["Path1"].attrs["marker"] = "retain" - file["params"].attrs["owner"] = "original" - file["unrelated/value"] = np.arange(3) - before = _snapshot(destination) - entries = ( - original.sources if owner == "Source" - else original.user_rois - ) - setattr(entries[1], field, np.ones(shape)) - with self.assertRaisesRegex( - ValueError, rf"{owner}\.{field}\[1\].*shape" - ): - write_summary(ExperimentSummary( - paths=[original], params={"operator": "new"} - ), destination) - after = _snapshot(destination) - self.assertEqual(after.keys(), before.keys()) - for key in before: - np.testing.assert_equal(after[key], before[key]) - - def test_empty_optional_template_shapes_are_validated(self): - """Zero-sized optional tensors must still have the right axes.""" - for key in ("dF_denoised", "events", "Fsvd"): - with self.subTest(key=key), TemporaryDirectory() as directory: - destination = Path(directory) / "summary.h5" - write_summary( - ExperimentSummary(paths=[_path(0, 0)]), destination - ) - original = read_summary(destination).paths[0] - before = _snapshot(destination) - templates = ( - original._empty_roi_arrays if key == "Fsvd" - else original._empty_source_arrays - ) - templates[key] = np.empty((0, 2, 8)) - with self.assertRaisesRegex(ValueError, rf"{key}.*shape"): - write_summary( - ExperimentSummary(paths=[original]), destination - ) - after = _snapshot(destination) - self.assertEqual(after.keys(), before.keys()) - for name in before: - np.testing.assert_equal(after[name], before[name]) - - def test_z_depths_are_always_column_vectors_without_mutation(self): - """Accept vector inputs while persisting the required (fastz, 1) rank.""" - for shape in ((3,), (1, 3), (3, 1)): - with self.subTest(shape=shape), TemporaryDirectory() as directory: - original = _path() - original.z_depths = original.z_depths.reshape(shape) - destination = Path(directory) / "summary.h5" - write_summary(ExperimentSummary(paths=[original]), destination) - with h5py.File(destination, "r") as file: - np.testing.assert_array_equal( - file["Path1/Z_depths"][()], [[2], [4], [8]] - ) - self.assertEqual(file["Path1/Z_depths"].dtype, np.int16) - self.assertEqual(original.z_depths.shape, shape) - - -if __name__ == "__main__": - unittest.main() \ No newline at end of file diff --git a/tests/test_io.py b/tests/test_io.py deleted file mode 100644 index 931a5ad..0000000 --- a/tests/test_io.py +++ /dev/null @@ -1,94 +0,0 @@ -"""Tests for the io subpackage.""" - -import os -import tempfile -import unittest - -import h5py -import numpy as np - -from giant_python import io -from giant_python.extraction.band import geometry, trial_data -from giant_python.io import load_struct_h5, slap2 -from giant_python.io.hdf5 import _decode_one, _decode_strings - - -class TestDecodeHelpers(unittest.TestCase): - """_decode_one / _decode_strings handle bytes, str, and empty arrays.""" - - def test_decode_one(self): - """bytes decode via utf-8; non-bytes pass through str().""" - self.assertEqual(_decode_one(b"ab"), "ab") - self.assertEqual(_decode_one("cd"), "cd") - - def test_decode_empty_array(self): - """An empty array is returned unchanged (no vectorize crash).""" - out = _decode_strings(np.array([], dtype=object)) - self.assertEqual(out.size, 0) - - -class TestHdf5(unittest.TestCase): - """Tests for the generic HDF5 struct reader.""" - - def test_load_struct_h5_faithful(self): - """load_struct_h5 mirrors groups, decodes strings, honors row_major.""" - str_dt = h5py.string_dtype(encoding="utf-8") - with tempfile.TemporaryDirectory() as d: - path = os.path.join(d, "s.h5") - with h5py.File(path, "w") as f: - # Column-major (row_major absent) -> 2-D axes transposed. - f.create_dataset("name", data="hello", dtype=str_dt) - f.create_dataset("mat", data=np.array([[1, 2, 3], [4, 5, 6]])) - grp = f.create_group("inner") - grp.create_dataset( - "labels", - data=np.array([["a", "b"]], dtype=object), - dtype=str_dt, - ) - out = load_struct_h5(path) - self.assertEqual(out["name"], "hello") - # (2, 3) written column-major reads back transposed to (3, 2). - np.testing.assert_array_equal( - out["mat"], np.array([[1, 4], [2, 5], [3, 6]]) - ) - np.testing.assert_array_equal( - out["inner"]["labels"], np.array([["a"], ["b"]], dtype=object) - ) - - def test_load_struct_h5_row_major(self): - """A row_major file keeps its on-disk orientation.""" - with tempfile.TemporaryDirectory() as d: - path = os.path.join(d, "s.h5") - with h5py.File(path, "w") as f: - f.create_dataset("row_major", data=1) - f.create_dataset("mat", data=np.array([[1, 2, 3]])) - out = load_struct_h5(path) - np.testing.assert_array_equal(out["mat"], np.array([[1, 2, 3]])) - - -class TestSlap2(unittest.TestCase): - """Tests for SLAP2 reader and band reduction ownership.""" - - def test_ref_pixs_to_drc(self): - """Reference-pixel conversion belongs to band geometry, not IO.""" - self.assertTrue(callable(geometry.ref_pixs_to_drc)) - self.assertEqual( - geometry.ref_pixs_to_drc.__module__, geometry.__name__ - ) - for module in (io, slap2): - self.assertFalse(hasattr(module, "ref_pixs_to_drc")) - self.assertNotIn("ref_pixs_to_drc", io.__all__) - - def test_read_band_trial_data(self): - """Band reduction remains canonical without an IO compatibility API.""" - self.assertTrue(callable(trial_data.read_band_trial_data)) - self.assertEqual( - trial_data.read_band_trial_data.__module__, trial_data.__name__ - ) - for module in (io, slap2): - self.assertFalse(hasattr(module, "read_band_trial_data")) - self.assertNotIn("read_band_trial_data", io.__all__) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_io_architecture.py b/tests/test_io_architecture.py deleted file mode 100644 index efb5f1e..0000000 --- a/tests/test_io_architecture.py +++ /dev/null @@ -1,281 +0,0 @@ -"""IO ownership, faithful decoding, and loaded-TrialTable regression tests.""" - -import ast -import sys -import tempfile -import unittest -from pathlib import Path -from types import ModuleType -from unittest.mock import Mock, patch - -import h5py -import numpy as np - -from giant_python.extraction.band import inputs, result_assembly -from giant_python.io import experiment_summary, hdf5, slap2, trial_table -from giant_python.io.annotations import ( - read_annotations_h5, - save_annotations_h5, -) -from giant_python.io.tiff import read_tiff -from giant_python.io.trial_table import read_trial_table -from giant_python.models.trial_table import Slap2Info, TrialTable - - -class TestSharedDecoder(unittest.TestCase): - """There is one HDF5 decoder and it preserves the storage convention.""" - - def test_removed_aliases_are_unavailable(self): - """There is one public spelling for each raw/schema operation.""" - from giant_python import io - - for owner, names in ( - ( - hdf5, - ( - "load_struct_from_h5", - "_h5_is_string_dataset", - "_decode_h5_strings", - "_read_h5_dataset", - "_read_h5_group", - ), - ), - (slap2, ("read_alignment_data_h5",)), - (trial_table, ("load_trial_table",)), - ( - experiment_summary, - ( - "write_experiment_summary", - "read_experiment_summary", - "path_summary_from_assembled", - "path_summary_to_assembled", - ), - ), - ( - result_assembly, - ("assemble_path_outputs", "build_experiment_summary"), - ), - (io, ("load_struct_from_h5", "write_experiment_summary")), - ): - for name in names: - with self.subTest(owner=owner.__name__, name=name): - self.assertFalse(hasattr(owner, name)) - self.assertIs(inputs.load_struct_h5, hdf5.load_struct_h5) - self.assertIs( - inputs.read_alignment_data_h5, slap2.load_alignment_data_h5 - ) - - def test_layout_flags_and_string_grids(self): - """Missing/zero flags reverse 3D axes; rank-two vectors survive.""" - cube = np.arange(24).reshape(2, 3, 4) - labels = np.array([["a", "b", "c"], ["d", "e", "f"]], dtype=object) - with tempfile.TemporaryDirectory() as dr: - for flag in (None, 0, 1): - with self.subTest(flag=flag): - path = Path(dr) / "layout.h5" - with h5py.File(path, "w") as f: - if flag is not None: - f["row_major"] = flag - f["cube"] = cube - f["vector"] = np.arange(3).reshape(1, 3) - f.create_dataset( - "labels", data=labels, dtype=h5py.string_dtype() - ) - decoded = hdf5.load_struct_h5(path) - np.testing.assert_array_equal( - decoded["cube"], cube if flag else cube.T - ) - np.testing.assert_array_equal( - decoded["labels"], labels if flag else labels.T - ) - self.assertEqual( - decoded["vector"].shape, (1, 3) if flag else (3, 1) - ) - - def test_raw_alignment_sign_and_private_adapter(self): - """Raw displacement stays unchanged; the backend negates offsets.""" - saved = np.array([1.25, -2.5, np.nan]) - with tempfile.TemporaryDirectory() as dr: - path = Path(dr) / "alignment.h5" - with h5py.File(path, "w") as f: - f["row_major"] = 1 - f["DSframes"] = np.array([[1, 5, 9]]) - f["numChannels"] = 2 - f["alignHz"] = 10.0 - for key in ("motionDSr", "motionDSc", "motionDSz"): - f[key] = saved.reshape(-1, 1) - f.create_group("slap2")["onlineMotionXshift"] = saved.reshape( - 1, -1 - ) - raw = slap2.load_alignment_data_h5(path) - with patch.object( - inputs, "read_alignment_data_h5", return_value=raw - ): - normalized = inputs.load_alignment_data_h5(path) - for key in ("motionDSr", "motionDSc", "motionDSz"): - np.testing.assert_array_equal(raw[key], saved) - np.testing.assert_array_equal(normalized[key], -saved) - self.assertFalse(np.shares_memory(raw[key], normalized[key])) - self.assertIs(normalized["onlineXshift"], raw["onlineXshift"]) - self.assertIsNone(normalized["onlineYshift"]) - - -class TestTrialTableBoundary(unittest.TestCase): - """Models retain raw metadata; backend views do path/file preparation.""" - - def test_in_memory_table_is_never_reloaded(self): - """A loaded model keeps identity without reconstructing a filename.""" - table = TrialTable( - datadr=Path("raw"), - savedr=Path("results"), - filename=np.array([["trial.dat"]], dtype=object), - slap2_info=Slap2Info(first_line=[[1]], last_line=[[9]]), - motion_correction={"fn_adata": np.array([["trial.h5"]])}, - source_path=Path("unusual-name.h5"), - ) - with patch.object( - TrialTable, "from_h5", side_effect=AssertionError("reload") - ): - view = inputs.load_trial_table(table) - self.assertIs(view["trial_table"], table) - self.assertEqual(view["source_path"], Path("unusual-name.h5")) - self.assertEqual( - view["fn_adata"][0, 0], - str(Path("results/motion_correction/trial.h5")), - ) - self.assertNotIn("keep_trials", view) - self.assertNotIn("align_hz", view) - - def test_model_codec_records_source_without_probing(self): - """Noncanonical table filenames survive the IO-model delegation.""" - with tempfile.TemporaryDirectory() as dr: - path = Path(dr) / "not-the-standard-name.h5" - with h5py.File(path, "w") as f: - f["row_major"] = 1 - f["datadr"] = "not-a-real-directory" - f.create_dataset( - "filename", data=[["x.dat"]], dtype=h5py.string_dtype() - ) - f["true_trial_ix"] = np.array([[4]]) - f.create_group("slap2_info")["first_line"] = np.array([[1]]) - table = TrialTable.from_h5(path) - self.assertEqual(table.source_path, path.absolute()) - self.assertEqual(table.filename.shape, (1, 1)) - np.testing.assert_array_equal(table.true_trial_ix, [[4]]) - with h5py.File(path, "r") as f: - self.assertNotIn("source_path", f) - self.assertEqual( - read_trial_table(path).source_path, table.source_path - ) - - -class TestRawAnnotations(unittest.TestCase): - """Raw annotations are readable without any reference-stack geometry.""" - - def test_provenance_overlap_and_empty_path(self): - """Masks and provenance survive; empty Path groups are explicit.""" - mask = np.ones((2, 3, 4), dtype=bool) - with tempfile.TemporaryDirectory() as dr: - file = save_annotations_h5( - dr, - {"DMD1": [{"label": "A"}, {"label": "B"}]}, - {"DMD1": [mask, mask]}, - 2, - ref_files={"DMD1": str(Path(dr) / "reference.tif")}, - ) - raw = read_annotations_h5(file) - self.assertEqual(raw["Path1"]["fn"], "reference.tif") - self.assertEqual(int(raw["Path2"]["n_rois"]), 0) - np.testing.assert_array_equal(raw["Path1"]["roi_000"]["mask"], mask) - np.testing.assert_array_equal(raw["Path1"]["roi_001"]["mask"], mask) - - -class TestRawAcquisitionSeams(unittest.TestCase): - """Raw reader construction and TIFF reads preserve native contracts.""" - - def test_open_preserves_vendor_selection_and_caller_ownership(self): - """Opening reads/closes nothing; CYCLE names use MultiDataFiles.""" - vendor = ModuleType("slap2_utils") - single, multiple = Mock(), Mock() - vendor.DataFile = Mock(return_value=single) - vendor.MultiDataFiles = Mock(return_value=multiple) - with patch.dict(sys.modules, {"slap2_utils": vendor}): - with patch("importlib.reload", return_value=vendor) as reload: - self.assertIs(slap2.open_slap2_file("trial.dat"), single) - self.assertIs( - slap2.open_slap2_file("trial_CYCLE001.dat"), multiple - ) - self.assertEqual(reload.call_count, 2) - vendor.DataFile.assert_called_once_with("trial.dat") - vendor.MultiDataFiles.assert_called_once_with("trial_CYCLE001.dat") - single.getLineData.assert_not_called() - multiple.getLineData.assert_not_called() - single.close.assert_not_called() - multiple.close.assert_not_called() - - def test_native_tiff_shape_dtype_and_intensity(self): - """Shared TIFF decoding does not perform band reference scaling.""" - import tifffile - - array = np.arange(60, dtype=np.uint16).reshape(3, 4, 5) - with tempfile.TemporaryDirectory() as dr: - path = Path(dr) / "reference.tif" - tifffile.imwrite(path, array, photometric="minisblack") - loaded = read_tiff(path) - self.assertEqual(loaded.dtype, array.dtype) - np.testing.assert_array_equal(loaded, array) - - -class TestImportOwnership(unittest.TestCase): - """Source-level import boundaries remain independent of heavy backends.""" - - def test_io_and_models_do_not_import_band_or_toolkits(self): - """Shared readers and models do not import band/numerics/GUI.""" - root = Path(__file__).resolve().parents[1] / "src/giant_python" - paths = list((root / "io").glob("*.py")) - paths += [ - root / "models/experiment_summary.py", - root / "models/trial_table.py", - ] - for path in paths: - with self.subTest(path=path.name): - tree = ast.parse(path.read_text(encoding="utf-8")) - for node in ast.walk(tree): - if isinstance(node, ast.ImportFrom): - module = node.module or "" - self.assertFalse( - any( - part in module.split(".") - for part in ( - "bandsilo", - "extraction", - "torch", - "cv2", - "tkinter", - ) - ) - ) - - def test_numerical_assembly_constructs_models_without_io(self): - """Results may depend on models/numerics, never persistence or GUI.""" - tree = ast.parse( - Path(result_assembly.__file__).read_text(encoding="utf-8") - ) - forbidden = {"io", "h5py", "workflow", "gui", "bandsilo"} - for node in ast.walk(tree): - if isinstance(node, ast.ImportFrom): - self.assertFalse( - forbidden.intersection((node.module or "").split(".")) - ) - elif isinstance(node, ast.Import): - for alias in node.names: - self.assertFalse( - forbidden.intersection(alias.name.split(".")) - ) - public_functions = { - node.name - for node in tree.body - if isinstance(node, ast.FunctionDef) - and not node.name.startswith("_") - } - self.assertEqual(public_functions, {"assemble_path_summary"}) diff --git a/tests/test_io_edges.py b/tests/test_io_edges.py deleted file mode 100644 index 9984c0e..0000000 --- a/tests/test_io_edges.py +++ /dev/null @@ -1,523 +0,0 @@ -"""Regression coverage for schema validation and public IO/model seams.""" - -import importlib -import sys -import unittest -from dataclasses import fields, replace -from pathlib import Path -from tempfile import TemporaryDirectory -from unittest.mock import patch - -import h5py -import numpy as np - -from giant_python.extraction.band import annotation, inputs -from giant_python.io.experiment_summary import read_summary, write_summary -from giant_python.io.hdf5 import write_dict_to_h5group -from giant_python.models.experiment_summary import ( - FRAME_INFO_KEYS, - ExperimentSummary, - FrameInfo, - PathSummary, - Source, - UserRoi, - Visualizations, -) -from giant_python.models.params import ( - AnnotationOptions, - BandSiloParams, - ExecutionOptions, -) -from giant_python.models.trial_table import Slap2Info - - -def _empty_source_path(**kwargs): - """Provide empty band tensors with explicit rank/dtype templates.""" - values = { - "z_depths": np.array([[1], [2]], dtype=np.int32), - "global_f": np.ones((2, 5), dtype=np.float32), - "frame_info": FrameInfo( - **{key: np.zeros((5, 1)) for key in FRAME_INFO_KEYS} - ), - "visualizations": Visualizations( - mean_im=np.ones((2, 2, 3, 4), dtype=np.float32), - act_im=np.ones((2, 3, 4), dtype=np.float32), - act_im_peaks=np.empty((0, 3), dtype=np.int32), - ), - **kwargs, - } - return PathSummary( - _empty_source_arrays={ - "profiles": np.empty((0, 2, 3, 4), dtype=np.float32), - "coords": np.empty((0, 3), dtype=np.int32), - "dF_ls": np.empty((0, 2, 5), dtype=np.float32), - "F0": np.empty((0, 2, 5), dtype=np.float32), - "SNR": np.empty((0, 1), dtype=np.float32), - }, - **values, - ) - - -def _roundtrip(path): - """Exercise public model persistence in a real temporary HDF5 file.""" - with TemporaryDirectory() as directory: - destination = Path(directory) / "summary.h5" - write_summary(ExperimentSummary(paths=[path]), destination) - return read_summary(destination).paths[0] - - -class TestSummaryValidation(unittest.TestCase): - """Invalid models fail explicitly rather than silently losing ROI data.""" - - def test_missing_arrays_preserve_existing_file_groups(self): - """Report missing arrays by field/index before replacing the path.""" - source = Source( - profile=np.ones((1, 2, 3), dtype=np.float32), - coords=np.array([0, 1, 2], dtype=np.int32), - df_ls=np.ones((1, 4), dtype=np.float32), - f0=np.ones((1, 4), dtype=np.float32), - snr=2.0, - ) - roi = UserRoi( - label="retained μ", - mask=np.ones((1, 2, 3), dtype=bool), - f=np.ones((1, 4), dtype=np.float32), - ) - shifts = np.array([[1.5, -2.0, np.nan]], dtype=np.float32) - with TemporaryDirectory() as directory: - destination = Path(directory) / "summary.h5" - with h5py.File(destination, "w") as file: - file["row_major"] = 1 - file.create_group("params").attrs["owner"] = "original" - for name in ("Path1", "Path2"): - group = file.create_group(name) - group.attrs["marker"] = name - group.create_group("frame_info").create_dataset( - "offlineXshifts", data=shifts - ) - for owner, field in ( - ("Source", "profile"), - ("Source", "coords"), - ("Source", "df_ls"), - ("Source", "f0"), - ("UserRoi", "mask"), - ("UserRoi", "f"), - ): - with self.subTest(owner=owner, field=field): - path = PathSummary( - path_index=1, - sources=[source, replace(source)], - user_rois=[roi, replace(roi)], - annotation_enabled=True, - ) - entry = ( - path.sources[1] - if owner == "Source" - else path.user_rois[1] - ) - setattr(entry, field, None) - message = rf"{owner}\.{field}\[1\] is required; got None" - with self.assertRaisesRegex(ValueError, message): - write_summary( - ExperimentSummary(paths=[path]), destination - ) - with h5py.File(destination, "r") as file: - self.assertEqual( - set(file), - {"row_major", "params", "Path1", "Path2"}, - ) - self.assertEqual(file["row_major"][()], 1) - self.assertEqual( - file["params"].attrs["owner"], "original" - ) - self.assertEqual(set(file["params"]), set()) - for name in ("Path1", "Path2"): - group = file[name] - self.assertEqual(group.attrs["marker"], name) - self.assertEqual(set(group), {"frame_info"}) - self.assertEqual( - set(group["frame_info"]), {"offlineXshifts"} - ) - saved = group["frame_info/offlineXshifts"] - np.testing.assert_array_equal(saved[()], shifts) - self.assertEqual(saved.dtype, shifts.dtype) - - def test_sources_without_shape_templates_are_rejected(self): - """Empty source lists need geometry that cannot be inferred.""" - with self.assertRaisesRegex(ValueError, "shape templates"): - _roundtrip(PathSummary()) - - def test_disabled_annotations_reject_supplied_rois(self): - """Callers must explicitly enable serialization of supplied ROIs.""" - path = _empty_source_path(user_rois=[UserRoi(label="keep me")]) - with self.assertRaisesRegex(ValueError, "annotation_enabled=True"): - _roundtrip(path) - self.assertEqual(path.user_rois[0].label, "keep me") - self.assertFalse(path.annotation_enabled) - - def test_direct_empty_rois_infer_layout_and_preserve_dtype(self): - """Enabled empty ROIs infer mask/trace geometry without mutation.""" - mean = np.ones((2, 2, 3, 4), dtype=np.float32) - global_f = np.ones((2, 5), dtype=np.float64) - path = _empty_source_path( - annotation_enabled=True, - visualizations=Visualizations( - mean_im=mean, - act_im=np.ones((2, 3, 4)), - act_im_peaks=np.empty((0, 3)), - ), - global_f=global_f, - ) - out = _roundtrip(path) - rois = out._empty_roi_arrays - self.assertEqual(rois["mask"].shape, (0, 2, 3, 4)) - self.assertEqual(rois["mask"].dtype, np.dtype(bool)) - self.assertEqual(rois["F"].shape, (0, 2, 5)) - self.assertEqual(rois["F"].dtype, global_f.dtype) - self.assertEqual(out.user_rois, []) - self.assertIs(path.visualizations.mean_im, mean) - self.assertIs(path.global_f, global_f) - np.testing.assert_array_equal(out.visualizations.mean_im, mean) - np.testing.assert_array_equal(out.global_f, global_f) - self.assertEqual(path._empty_roi_arrays, {}) - - def test_empty_rois_require_both_geometry_ranks(self): - """Neither malformed image geometry nor trace geometry is guessed.""" - for mean_shape, trace_shape in ( - ((2, 3, 4), (2, 5)), - ((2, 2, 3, 4), (5,)), - ): - with self.subTest(mean=mean_shape, traces=trace_shape): - path = _empty_source_path( - annotation_enabled=True, - visualizations=Visualizations( - mean_im=np.zeros(mean_shape) - ), - global_f=np.zeros(trace_shape), - ) - with self.assertRaisesRegex(ValueError, "geometry"): - _roundtrip(path) - - def test_duplicate_path_ids_fail_before_opening_destination(self): - """Duplicate identities cannot replace data in a destination file.""" - summary = ExperimentSummary( - paths=[PathSummary(path_index=3), PathSummary(path_index=3)] - ) - with patch("giant_python.io.experiment_summary.h5py.File") as opener: - with self.assertRaisesRegex(ValueError, "duplicate path_index"): - write_summary(summary, "must-not-open.h5") - opener.assert_not_called() - - def test_model_identity_validation_and_empty_fields(self): - """Path IDs are nonnegative integers; empty summaries stay empty.""" - for value in (-1, 1.5, "1", None): - with self.subTest(value=value): - with self.assertRaisesRegex(ValueError, "nonnegative"): - PathSummary(path_index=value) - self.assertEqual(PathSummary(path_index=np.int64(2)).path_index, 2) - summary = ExperimentSummary() - self.assertEqual(summary.paths, []) - self.assertIsNone(summary.params) - self.assertEqual( - {item.name for item in fields(summary)}, {"paths", "params"} - ) - info = FrameInfo() - self.assertEqual(tuple(info.to_dict()), FRAME_INFO_KEYS) - self.assertTrue( - all(value is None for value in info.to_dict().values()) - ) - - def test_path_fields_share_original_arrays(self): - """Explicit path access preserves array identity, dtype and rank.""" - path = PathSummary( - global_f=np.ones((2, 5), dtype=np.float32), - z_depths=np.array([[1], [3]], dtype=np.int16), - ) - summary = ExperimentSummary(paths=[path]) - self.assertIs(summary.paths[0].global_f, path.global_f) - self.assertIs(summary.paths[0].z_depths, path.z_depths) - self.assertIs(summary.paths[0].visualizations, path.visualizations) - - def test_append_retains_layout_flags_and_unrelated_groups(self): - """Replacement preserves root metadata, including MATLAB flags.""" - for layout in (0, 1): - with ( - self.subTest(layout=layout), - TemporaryDirectory() as directory, - ): - destination = Path(directory) / "summary.h5" - with h5py.File(destination, "w") as file: - file["row_major"] = np.uint8(layout) - file["coords_zero_indexed"] = np.uint8(0) - file["params/operator"] = "original μ" - file["Path1/obsolete"] = 5 - file["Path12/retained"] = np.array([[2, 7]], np.int16) - file["unrelated/labels"] = "retained μ" - path = _empty_source_path() - write_summary( - ExperimentSummary( - paths=[path], params={"operator": "replacement"} - ), - destination, - ) - with h5py.File(destination, "r") as file: - self.assertEqual(file["row_major"][()], layout) - self.assertEqual(file["row_major"].dtype, np.uint8) - self.assertEqual(file["coords_zero_indexed"][()], 0) - self.assertEqual( - file["params/operator"].asstr()[()], "original μ" - ) - self.assertEqual( - file["unrelated/labels"].asstr()[()], "retained μ" - ) - self.assertNotIn("obsolete", file["Path1"]) - np.testing.assert_array_equal( - file["Path1/Z_depths"], path.z_depths - ) - np.testing.assert_array_equal( - file["Path12/retained"], [[2, 7]] - ) - self.assertEqual(file["Path12/retained"].dtype, np.int16) - - -class TestBandInputValidation(unittest.TestCase): - """Missing lookup/reference groups fail at the DMD input boundary.""" - - def test_missing_lookup_groups_identify_requested_dmd(self): - """Absent or None aliases fail after any preceding valid DMD group.""" - valid = { - "allSuperPixelIDs": np.array([1]), - "sparseMaskInds": np.array([[1, 2]]), - "fastZ2RefZ": np.array([1]), - } - for missing in ({}, {"Path2": None}, {"DMD2": None}): - with self.subTest(missing=missing): - raw = {"Path1": valid, **missing} - with patch.object(inputs, "load_struct_h5", return_value=raw): - with self.assertRaisesRegex( - ValueError, "Missing lookup-table group Path2 or DMD2" - ): - inputs.load_lookup_table("lookup.h5", 2) - - def test_missing_reference_groups_fail_before_file_discovery(self): - """Distinguish missing metadata from a missing reference TIFF.""" - for missing in ({}, {"Path2": None}, {"DMD2": None}): - with self.subTest(missing=missing): - raw = {"Path1": {"channels": np.array([1])}, **missing} - with ( - patch.object(inputs, "find_reference_file") as find, - patch.object(inputs, "read_tiff") as read, - ): - with self.assertRaisesRegex( - ValueError, - "Missing reference-stack group Path2 or DMD2", - ): - inputs.load_reference_stack("raw", raw, 1) - find.assert_not_called() - read.assert_not_called() - - -class TestParameterWriterEdges(unittest.TestCase): - """Arrays and Python sequences retain their persisted string policies.""" - - def test_workflow_parameters_keep_numeric_storage(self): - """Canonical Python params retain scalar types and UTF-8 labels.""" - summary = ExperimentSummary( - params={ - "numChannels": 2, - "analyzeHz": 100.0, - "draw_user_rois": True, - "operator": "operator μ", - } - ) - with TemporaryDirectory() as directory: - destination = Path(directory) / "summary.h5" - write_summary(summary, destination) - with h5py.File(destination, "r") as file: - expected = { - "numChannels": (2, np.int64), - "analyzeHz": (100.0, np.float64), - "draw_user_rois": (1, np.uint8), - } - for name, (value, dtype) in expected.items(): - dataset = file[f"params/{name}"] - self.assertEqual(dataset.shape, ()) - self.assertEqual(dataset.dtype, dtype) - self.assertEqual(dataset[()], value) - self.assertEqual( - file["params/operator"].asstr()[()], "operator μ" - ) - - def test_preserved_strings_and_sequences(self): - """UTF-8 grids retain shape; sequence bytes keep their str spelling.""" - values = { - "unicode": np.array([["μ", ""], ["é", "last"]]), - "bytes": np.array([[b"first", b"\xff"]]), - "objects": np.array([["μ", b"tail"]], dtype=object), - "scalar": np.str_("μ"), - "empty": np.empty((0, 2), dtype="U1"), - "nested": {"flag": np.uint8(1)}, - } - with h5py.File( - "parameter-edges.h5", "w", driver="core", backing_store=False - ) as file: - write_dict_to_h5group(file, values) - np.testing.assert_array_equal( - file["unicode"].asstr()[()], values["unicode"] - ) - np.testing.assert_array_equal( - file["bytes"].asstr()[()], [["first", "�"]] - ) - np.testing.assert_array_equal( - file["objects"].asstr()[()], [["μ", "tail"]] - ) - self.assertEqual(file["scalar"].asstr()[()], "μ") - self.assertEqual(file["empty"].shape, (0, 2)) - self.assertEqual(file["nested/flag"].dtype, np.dtype("uint8")) - legacy = file.create_group("legacy") - write_dict_to_h5group( - legacy, - { - "bytes": [b"first", b"last"], - "numbers": (1, 2), - "empty": [], - "none": None, - }, - ) - np.testing.assert_array_equal( - legacy["bytes"].asstr()[()], ["b'first'", "b'last'"] - ) - np.testing.assert_array_equal(legacy["numbers"][()], [1, 2]) - self.assertNotIn("empty", legacy) - self.assertNotIn("none", legacy) - - -class TestModelAndImportBoundaries(unittest.TestCase): - """Canonical model readers and public imports keep their ownership.""" - - def test_trial_table_metadata_adapter(self): - """Raw metadata decoding filters unknown fields without copying.""" - from giant_python.io.trial_table import slap2_info_from_raw - - first_line = np.array([[1, 9]], dtype=np.int32) - raw = {"first_line": first_line, "unknown": "ignored"} - info = slap2_info_from_raw(raw) - self.assertIsInstance(info, Slap2Info) - self.assertIs(info.first_line, first_line) - self.assertFalse(hasattr(info, "unknown")) - self.assertIsNone(slap2_info_from_raw(None)) - self.assertIsNone(slap2_info_from_raw({})) - - def test_lazy_io_exports_and_unknown_attribute(self): - """Schema exports resolve and cache canonical functions lazily.""" - from giant_python import io - - importlib.reload(io) - for name, owner in io._SCHEMA_EXPORTS.items(): - with self.subTest(name=name): - expected = getattr( - importlib.import_module(owner, io.__name__), name - ) - self.assertIs(io.__getattr__(name), expected) - self.assertIs(getattr(io, name), expected) - self.assertIn(name, io.__all__) - with self.assertRaisesRegex(AttributeError, "unknown_codec"): - io.__getattr__("unknown_codec") - - def test_annotation_and_options_import_without_toolkits(self): - """Headless orchestration and scoped option metadata need no GUI.""" - with patch.dict(sys.modules, {"tkinter": None, "cv2": None}): - importlib.reload(annotation) - self.assertTrue(callable(annotation.annotate_band_rois)) - science = {item.name for item in fields(BandSiloParams)} - execution = {item.name for item in fields(ExecutionOptions)} - annotations = {item.name for item in fields(AnnotationOptions)} - self.assertFalse(science & execution) - self.assertFalse(science & annotations) - self.assertFalse(execution & annotations) - - -class TestAnnotationGeometryAssembly(unittest.TestCase): - """Per-DMD geometry uses each path's first kept trial and provenance.""" - - def test_geometry_uses_first_kept_trial_for_each_dmd(self): - """Distinct motion offsets and references stay associated by DMD.""" - references = [np.ones((1, 2, 5, 6)), np.full((1, 2, 5, 6), 2.0)] - table = { - "datadr": Path("raw"), - "ref_stack": {"metadata": "retained"}, - "n_dmds": 2, - "keep_trials": np.array([[False, True], [True, False]]), - "fn_adata": np.array([["skip-a", "a.h5"], ["b.h5", "skip-b"]]), - } - lookup = { - "allSuperPixelIDs": {"DMD1": [11], "DMD2": [22]}, - "sparseMaskInds": {"DMD1": [33], "DMD2": [44]}, - "fastZ2RefZ": {"DMD1": np.array([1, 2]), "DMD2": np.array([1, 2])}, - } - motion = [ - {"motionDSr": [1.0], "motionDSc": [-1.0], "motionDSz": [0.0]}, - {"motionDSr": [-1.0], "motionDSc": [1.0], "motionDSz": [0.0]}, - ] - with ( - patch.object( - annotation.inputs, - "load_reference_stack", - side_effect=[ - (references[0], None, "ref-a.tif"), - (references[1], None, "ref-b.tif"), - ], - ) as load_reference, - patch.object( - annotation.geo, - "build_subsample_matrix_inds", - return_value=np.array([[6], [36]]), - ) as subsample, - patch.object( - annotation, "load_alignment_data_h5", side_effect=motion - ) as load_alignment, - ): - geometry, provenance = annotation.build_user_roi_geometry( - table, lookup - ) - self.assertEqual( - provenance, {"DMD1": "ref-a.tif", "DMD2": "ref-b.tif"} - ) - self.assertEqual( - [call.args[0] for call in load_alignment.call_args_list], - ["a.h5", "b.h5"], - ) - for index, key in enumerate(("DMD1", "DMD2")): - with self.subTest(path=key): - self.assertIs(geometry[key]["ref"], references[index]) - np.testing.assert_array_equal(geometry[key]["z_map"], [0, 1]) - np.testing.assert_array_equal( - geometry[key]["sp_rows"], [2, 2] if index == 0 else [0, 0] - ) - np.testing.assert_array_equal( - geometry[key]["sp_cols"], [0, 0] if index == 0 else [2, 2] - ) - self.assertEqual(np.count_nonzero(geometry[key]["sp_mask"]), 2) - self.assertEqual( - load_reference.call_args_list[index].args, - ("raw", table["ref_stack"], index), - ) - self.assertEqual( - subsample.call_args_list[index].args, - ( - lookup["allSuperPixelIDs"][key], - lookup["sparseMaskInds"][key], - ), - ) - np.testing.assert_array_equal( - table["keep_trials"], [[False, True], [True, False]] - ) - - def test_annotation_option_resolution_preserves_input(self): - """Annotation uses scoped option conversion without mutation.""" - original = AnnotationOptions(interactive=False) - _, _, resolved = annotation.resolve_band_options(annotations=original) - self.assertEqual(resolved, original) - self.assertIsNot(resolved, original) - self.assertFalse(resolved.interactive) - self.assertFalse(original.interactive) diff --git a/tests/test_models.py b/tests/test_models.py deleted file mode 100644 index cfc9ce4..0000000 --- a/tests/test_models.py +++ /dev/null @@ -1,165 +0,0 @@ -"""Tests for the models subpackage.""" - -import os -import tempfile -import unittest -from unittest import mock - -import h5py -import numpy as np - -from giant_python.models import ( - AnnotationOptions, - BandSiloParams, - ExperimentSummary, - FrameInfo, - PathSummary, - Slap2Info, - Source, - TrialTable, - UserRoi, - Visualizations, -) - - -class TestDataclasses(unittest.TestCase): - """Dataclasses should construct with sensible defaults.""" - - def test_trial_table_defaults(self): - """TrialTable constructs empty with no motion/source groups.""" - tt = TrialTable() - self.assertIsNone(tt.motion_correction) - self.assertIsNone(tt.source_extraction) - - def test_slap2_info_defaults(self): - """Slap2Info constructs with empty collections.""" - self.assertEqual(Slap2Info().ref_stack, {}) - - def test_experiment_summary_defaults(self): - """ExperimentSummary owns per-path results, not aggregate aliases.""" - summary = ExperimentSummary() - self.assertEqual(summary.paths, []) - path = PathSummary() - summary.paths.append(path) - self.assertIsInstance(summary.paths[0].visualizations, Visualizations) - self.assertIsInstance(summary.paths[0].frame_info, FrameInfo) - self.assertEqual(summary.paths[0].sources, []) - for name in ( - "sources", - "user_rois", - "visualizations", - "frame_info", - "global_f", - "z_depths", - ): - with self.subTest(name=name): - self.assertFalse(hasattr(summary, name)) - - def test_source_and_user_roi(self): - """Source and UserRoi construct with defaults.""" - self.assertIsNone(Source().snr) - self.assertEqual(UserRoi().type, "polygon") - - def test_param_models(self): - """Fixed algorithms and acquisition metadata are not parameters.""" - params = BandSiloParams() - for name in ( - "background_interpolation", - "num_channels", - "activity_channel", - ): - with self.subTest(name=name): - self.assertFalse(hasattr(params, name)) - - def test_band_value_params_have_concrete_defaults(self): - """Value params default to concrete (non-None) values.""" - p = BandSiloParams() - self.assertEqual(p.analyze_hz, 100.0) - # Seven is the current implementation default, not a new retuning. - self.assertEqual(p.peakth, 7.0) - # Only runtime-resolved annotation policy retains a None sentinel. - self.assertIsNone(AnnotationOptions().interactive) - - -class TestH5RoundTrip(unittest.TestCase): - """Implemented model codecs preserve data and delegate to shared IO.""" - - def test_trial_table_from_h5(self): - """from_h5 faithfully mirrors the trial_table.h5 group hierarchy.""" - str_dt = h5py.string_dtype(encoding="utf-8") - with tempfile.TemporaryDirectory() as d: - path = os.path.join(d, "trial_table.h5") - with h5py.File(path, "w") as f: - f.create_dataset("row_major", data=1) - f.create_dataset("datadr", data="/data", dtype=str_dt) - f.create_dataset("savedr", data="/res", dtype=str_dt) - f.create_dataset( - "filename", - data=np.array([["t1", "t2"]], dtype=object), - dtype=str_dt, - ) - si = f.create_group("slap2_info") - si.create_dataset("first_line", data=np.array([[1, 2]])) - si.create_dataset("last_line", data=np.array([[10, 20]])) - rs = si.create_group("ref_stack") - rs.create_group("Path1").create_dataset( - "channels", data=np.array([1, 2]) - ) - mc = f.create_group("motion_correction") - mc.create_dataset( - "fn_adata", - data=np.array([["a.h5", ""]], dtype=object), - dtype=str_dt, - ) - tt = TrialTable.from_h5(path) - self.assertEqual(str(tt.datadr), os.path.normpath("/data")) - self.assertEqual(str(tt.savedr), os.path.normpath("/res")) - self.assertIsInstance(tt.slap2_info, Slap2Info) - np.testing.assert_array_equal( - np.asarray(tt.slap2_info.first_line), np.array([[1, 2]]) - ) - self.assertIn("Path1", tt.slap2_info.ref_stack) - self.assertEqual(tt.motion_correction["fn_adata"][0, 0], "a.h5") - self.assertIsNone(tt.source_extraction) - - def test_trial_table_from_h5_non_slap2(self): - """A table without slap2_info yields slap2_info=None.""" - str_dt = h5py.string_dtype(encoding="utf-8") - with tempfile.TemporaryDirectory() as d: - path = os.path.join(d, "trial_table.h5") - with h5py.File(path, "w") as f: - f.create_dataset("row_major", data=1) - f.create_dataset("datadr", data="/data", dtype=str_dt) - f.create_dataset("savedr", data="/res", dtype=str_dt) - f.create_dataset( - "filename", - data=np.array([["t1"]], dtype=object), - dtype=str_dt, - ) - tt = TrialTable.from_h5(path) - self.assertIsNone(tt.slap2_info) - self.assertIsNone(tt.motion_correction) - - def test_experiment_summary_from_h5(self): - """ExperimentSummary.from_h5 delegates to the shared band codec.""" - expected = ExperimentSummary() - with mock.patch( - "giant_python.io.experiment_summary.read_summary", - return_value=expected, - ) as reader: - result = ExperimentSummary.from_h5("custom_summary.h5") - self.assertIs(result, expected) - reader.assert_called_once_with("custom_summary.h5") - - def test_experiment_summary_to_h5(self): - """ExperimentSummary.to_h5 delegates without rewriting the model.""" - summary = ExperimentSummary() - with mock.patch( - "giant_python.io.experiment_summary.write_summary" - ) as writer: - self.assertIsNone(summary.to_h5("custom_summary.h5")) - writer.assert_called_once_with(summary, "custom_summary.h5") - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_motion_convention.py b/tests/test_motion_convention.py deleted file mode 100644 index c7c3e5a..0000000 --- a/tests/test_motion_convention.py +++ /dev/null @@ -1,89 +0,0 @@ -"""Regression tests for MATLAB-compatible public offline displacement.""" - -import tempfile -import unittest -from pathlib import Path - -import h5py -import numpy as np - -from giant_python.extraction.band.inputs import load_alignment_data_h5 -from giant_python.extraction.band.result_assembly import assemble_path_summary -from giant_python.extraction.band.traces import _interp_trial_alignment -from giant_python.io.experiment_summary import write_summary -from giant_python.models.experiment_summary import ExperimentSummary -from tests.test_bandsilo_pipeline import _path_result - - -class TestMotionConvention(unittest.TestCase): - """Convert private reference offsets back to public displacement once.""" - - def test_alignment_to_summary_preserves_saved_displacement(self): - """Signed fractional motion survives loading, interpolation and IO.""" - with tempfile.TemporaryDirectory() as directory: - alignment = Path(directory) / "alignment.h5" - ds = np.array([1.0, 3.0, 5.0, 7.0]) - frames = np.arange(1.0, 8.0) - displacement = ( - np.array([1.5, -2.0, 0.0, 3.0]), - np.array([-3.0, 2.5, 1.0, -1.0]), - np.array([0.5, -0.5, 1.5, 0.0]), - ) - with h5py.File(alignment, "w") as f: - f["row_major"] = 1 - f["DSframes"] = ds - f["numChannels"] = 1 - f["alignHz"] = 10.0 - for axis, values in zip("rcz", displacement): - f[f"motionDS{axis}"] = values - for axis, value in zip("YXZ", (2, -3, 1)): - f[f"slap2/onlineMotion{axis}shift"] = np.full(4, value) - loaded = load_alignment_data_h5(alignment) - motion = _interp_trial_alignment( - frames, loaded, np.zeros((1, ds.size)) - ) - pr = _path_result() - trial = pr.trace_results[1]._replace( - frame_line_idxs=frames, - reference_offsets=motion[:3], - online_shifts=motion[4:], - ) - pr.trace_results = [trial] - original = [a.copy() for a in trial.reference_offsets] - path = assemble_path_summary(pr) - summary = ExperimentSummary(paths=[path]) - output = Path(directory) / "summary.h5" - write_summary(summary, output) - with h5py.File(output, "r") as f: - for axis, values in zip("YXZ", displacement): - key = f"offline{axis}shifts" - expected = np.interp(frames, ds, values)[:, None] - np.testing.assert_allclose( - getattr(summary.paths[0].frame_info, key), expected - ) - np.testing.assert_allclose( - f[f"Path1/frame_info/{key}"], expected - ) - for axis, value in zip("YXZ", (2, -3, 1)): - key = f"online{axis}shifts" - np.testing.assert_array_equal( - getattr(path.frame_info, key), np.full((7, 1), value) - ) - for before, after in zip(original, trial.reference_offsets): - np.testing.assert_array_equal(before, after) - - def test_output_negation_preserves_nan_and_input_arrays(self): - """NaNs and trial order survive without modifying numerical inputs.""" - pr = _path_result() - pr.trace_results[0].reference_offsets[0][0] = np.nan - original = np.concatenate( - [r.reference_offsets[0] for r in pr.trace_results] - ) - path = assemble_path_summary(pr) - np.testing.assert_array_equal( - path.frame_info.offlineYshifts.ravel(), -original - ) - np.testing.assert_array_equal( - np.concatenate([r.reference_offsets[0] for r in pr.trace_results]), - original, - ) diff --git a/tests/test_no_placeholders.py b/tests/test_no_placeholders.py deleted file mode 100644 index 73da82f..0000000 --- a/tests/test_no_placeholders.py +++ /dev/null @@ -1,335 +0,0 @@ -"""Prevent removed placeholder modules, APIs, and empty functions returning.""" - -import ast -import inspect -import unittest -from importlib import import_module -from importlib.util import find_spec -from pathlib import Path -from textwrap import dedent - -from giant_python.models import TrialTable -from giant_python.pipeline import Pipeline - -PACKAGE_ROOT = Path(__file__).resolve().parents[1] / "src" / "giant_python" - -REMOVED_MODULES = ( - "numerics.deconv", - "numerics.nmf", - "numerics.registration", - "numerics.variance", - "pipeline.base", - "pipeline.organize", - "pipeline.register", - "gui.draw_rois", - "models.alignment", -) -REMOVED_STAGES = ( - "Stage", - "TrialTableBuilder", - "MotionCorrector", - "MultiRoiRegistration", - "StripRegistration", - "BandRegistration", - "build_trial_table", - "build_trial_table_slap2", - "verify_files", - "multi_roi_registration", - "strip_registration", - "band_registration", - "silo", -) -REMOVED_KERNELS = ( - "deconvolve_trace", - "nmf_decompose", - "xcorr2_nans_weighted", - "xcorr2_nans", - "xcorr2_nans3d", - "dft_register_clipped", - "activity_image", - "interp_frame", -) -REMOVED_IO = ( - "save_struct_h5", - "get_online_motion", - "scanimagetiff_wrapper", - "scanimagetiff_data_wrapper", -) -REMOVED_EXPORTS = { - "giant_python": ( - *REMOVED_STAGES, - *REMOVED_KERNELS, - *REMOVED_IO, - "AlignParams", - "AlignmentData", - "DrawROIs", - ), - "giant_python.models": ("AlignParams", "AlignmentData"), - "giant_python.models.params": ("AlignParams",), - "giant_python.pipeline": REMOVED_STAGES, - "giant_python.pipeline.pipeline": (*REMOVED_STAGES, "AlignParams"), - "giant_python.pipeline.extract": ("silo",), - "giant_python.numerics": REMOVED_KERNELS, - "giant_python.numerics.interpolation": ("interp_frame",), - "giant_python.io": REMOVED_IO, - "giant_python.io.hdf5": ("save_struct_h5",), - "giant_python.io.slap2": ("get_online_motion",), - "giant_python.io.tiff": ( - "scanimagetiff_wrapper", - "scanimagetiff_data_wrapper", - ), - "giant_python.gui": ("DrawROIs", "annotate_rois"), -} - - -def _is_stub_function(node): - """Recognize functions containing only a docstring, pass, or ellipsis.""" - body = node.body[1:] if ast.get_docstring(node) is not None else node.body - return all( - isinstance(statement, ast.Pass) - or ( - isinstance(statement, ast.Expr) - and isinstance(statement.value, ast.Constant) - and statement.value.value is Ellipsis - ) - for statement in body - ) - - -def _placeholder_violations(tree, relative_path): - """Return line-numbered AST violations, exempting one exact protocol.""" - parents = { - child: parent - for parent in ast.walk(tree) - for child in ast.iter_child_nodes(parent) - } - violations = [] - for node in ast.walk(tree): - if ( - (isinstance(node, ast.Name) and node.id == "NotImplementedError") - or ( - isinstance(node, ast.Attribute) - and node.attr == "NotImplementedError" - ) - or ( - isinstance(node, ast.alias) - and node.name == "NotImplementedError" - ) - ): - violations.append((node.lineno, "NotImplementedError")) - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - if _is_stub_function(node): - parent = parents[node] - is_reader_contract = ( - relative_path == "io/slap2.py" - and isinstance(parent, ast.ClassDef) - and parent.name == "Slap2Reader" - and isinstance(parents[parent], ast.Module) - and any( - isinstance(base, ast.Name) and base.id == "Protocol" - for base in parent.bases - ) - and node.name == "getLineData" - and len(node.body) == 1 - and ast.get_docstring(node) is not None - ) - if not is_reader_contract: - violations.append((node.lineno, "stub: " + node.name)) - return violations - - -class TestRemovedPlaceholders(unittest.TestCase): - """Removal means physical absence and no discoverable replacement API.""" - - def test_deleted_modules_are_physically_absent(self): - """Neither source files nor same-name package shells may remain.""" - for module in REMOVED_MODULES: - path = PACKAGE_ROOT.joinpath(*module.split(".")) - with self.subTest(module=module): - self.assertFalse(path.with_suffix(".py").exists()) - self.assertFalse(path.exists()) - - def test_deleted_modules_cannot_be_discovered(self): - """Import discovery cannot find old modules or compatibility shims.""" - for module in REMOVED_MODULES: - with self.subTest(module=module): - self.assertIsNone(find_spec("giant_python." + module)) - - def test_removed_owner_and_package_exports_are_absent(self): - """Removed symbols cannot survive in owners, exports, or completion.""" - for module_name, names in REMOVED_EXPORTS.items(): - module = import_module(module_name) - for name in names: - with self.subTest(module=module_name, name=name): - self.assertFalse(hasattr(module, name)) - self.assertNotIn(name, getattr(module, "__all__", ())) - self.assertNotIn(name, dir(module)) - - def test_removed_model_and_pipeline_members_are_absent(self): - """Future serialization, stages, and unused session state are gone.""" - for owner in (TrialTable, TrialTable()): - with self.subTest(owner=owner): - self.assertFalse(hasattr(owner, "to_h5")) - for owner in (Pipeline, Pipeline()): - for name in ( - "from_config", - "organize", - "register", - "run_all", - "save_dir", - "align_params", - ): - with self.subTest(owner=owner, name=name): - self.assertFalse(hasattr(owner, name)) - - def test_constructor_exposes_only_supported_options(self): - """Science and routing remain distinct from keyword-only policies.""" - parameters = inspect.signature(Pipeline).parameters - self.assertEqual( - list(parameters), - ["microscope", "params", "execution", "annotations", "scan_mode"], - ) - for name in ("microscope", "params"): - self.assertEqual( - parameters[name].kind, inspect.Parameter.POSITIONAL_OR_KEYWORD - ) - for name in ("execution", "annotations", "scan_mode"): - self.assertEqual( - parameters[name].kind, inspect.Parameter.KEYWORD_ONLY - ) - - def test_removed_constructor_keywords_are_rejected(self): - """Direct and loaded-table constructors reject removed options.""" - table = TrialTable() - for name in ("save_dir", "align_params", "silo_params"): - with self.subTest(keyword=name): - with self.assertRaisesRegex(TypeError, name): - Pipeline(**{name: None}) - with self.assertRaisesRegex(TypeError, name): - Pipeline.from_trial_table(table, **{name: None}) - - def test_production_ast_contains_no_placeholders(self): - """Scan every source function, including private and nested helpers.""" - paths = sorted(PACKAGE_ROOT.rglob("*.py")) - self.assertTrue(paths) - for path in paths: - relative = path.relative_to(PACKAGE_ROOT).as_posix() - tree = ast.parse(path.read_text(encoding="utf-8-sig")) - with self.subTest(path=relative): - self.assertEqual(_placeholder_violations(tree, relative), []) - - def test_real_reader_protocol_is_retained(self): - """The used raw-reader typing declaration is not a removed stub API.""" - from giant_python.io import Slap2Reader - from giant_python.io.slap2 import Slap2Reader as Reader - - self.assertIs(Slap2Reader, Reader) - self.assertTrue(callable(Reader.getLineData)) - self.assertIn("getLineData", Reader.__dict__) - tree = ast.parse( - (PACKAGE_ROOT / "io" / "slap2.py").read_text(encoding="utf-8-sig") - ) - reader = next( - node - for node in tree.body - if isinstance(node, ast.ClassDef) and node.name == "Slap2Reader" - ) - method = next( - node - for node in reader.body - if isinstance(node, ast.FunctionDef) and node.name == "getLineData" - ) - self.assertTrue(_is_stub_function(method)) - self.assertEqual(len(method.body), 1) - self.assertIsNotNone(ast.get_docstring(method)) - self.assertEqual(_placeholder_violations(tree, "io/slap2.py"), []) - - -class TestPlaceholderGuard(unittest.TestCase): - """Exercise the AST guard so forbidden cases cannot silently escape it.""" - - def test_empty_function_variants_are_detected(self): - """Docstrings, pass, ellipsis, async methods, and nested stubs fail.""" - cases = ( - ("def stub(): pass", "stub"), - ("def stub(): ...", "stub"), - ('def stub(): "Only a docstring."', "stub"), - ('def stub():\n "Description."\n pass\n ...', "stub"), - ("async def stub(): ...", "stub"), - ("class Example:\n def method(self): pass", "method"), - ("def outer():\n def inner(): ...", "inner"), - ) - for source, name in cases: - with self.subTest(source=source): - violations = _placeholder_violations( - ast.parse(source), "example.py" - ) - self.assertEqual( - [reason for _, reason in violations], ["stub: " + name] - ) - - def test_not_implemented_error_forms_are_detected(self): - """Bare, constructed, qualified, and aliased error references fail.""" - for source in ( - "raise NotImplementedError", - "raise NotImplementedError('future')", - "raise builtins.NotImplementedError()", - "from builtins import NotImplementedError as FutureError", - ): - with self.subTest(source=source): - self.assertEqual( - _placeholder_violations(ast.parse(source), "example.py"), - [(1, "NotImplementedError")], - ) - - def test_real_functions_and_non_code_mentions_are_allowed(self): - """Ignore comments, strings, and pass within implemented functions.""" - source = dedent('''\ - # NotImplementedError is mentioned, not raised. - def implemented(value): - """NotImplementedError is just documentation here.""" - try: - return value[...] - except IndexError: - pass - return None - - async def asynchronous(): - return "NotImplementedError" - ''') - self.assertEqual( - _placeholder_violations(ast.parse(source), "example.py"), [] - ) - - def test_protocol_exception_is_exact(self): - """No other file, class, method, or empty body gets an exemption.""" - declaration = dedent('''\ - class Slap2Reader(Protocol): - def getLineData(self): - """Vendor reader typing contract.""" - ''') - self.assertEqual( - _placeholder_violations(ast.parse(declaration), "io/slap2.py"), [] - ) - cases = ( - (declaration, "other.py"), - (declaration.replace("Slap2Reader", "OtherReader"), "io/slap2.py"), - (declaration.replace("getLineData", "other"), "io/slap2.py"), - (declaration.replace("(Protocol)", ""), "io/slap2.py"), - ( - declaration.replace( - '"""Vendor reader typing contract."""', "pass" - ), - "io/slap2.py", - ), - (declaration + " ...\n", "io/slap2.py"), - ) - for source, path in cases: - with self.subTest(source=source, path=path): - self.assertEqual( - len(_placeholder_violations(ast.parse(source), path)), 1 - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_numeric_architecture.py b/tests/test_numeric_architecture.py deleted file mode 100644 index a53a54b..0000000 --- a/tests/test_numeric_architecture.py +++ /dev/null @@ -1,296 +0,0 @@ -"""Canonical kernel ownership, dependency direction, and retired APIs.""" - -import ast -import importlib -import importlib.util -import unittest -from pathlib import Path - -PACKAGE_ROOT = Path(__file__).resolve().parents[1] / "src" / "giant_python" - -# Each kernel has one implementation, in its canonical owning module. -OWNERS = { - "numerics.baseline": ( - "_movmean_nan", - "_compute_f0_column", - "compute_f0", - "assemble_dff", - ), - "numerics.peaks": ( - "gaussian_peaks_integrated", - "_gaussian_peaks_integrated_val_jac", - "_lsq_curvefit", - "_fit_isotropic_then_anisotropic", - "_make_bounds", - "_peak_mask", - "_buffer_mask", - "_process_cc_new_peak", - "_refine_residual_peaks", - "detect_peaks_2d", - "get_act_im_peaks", - ), - "numerics.interpolation": ("nearest_interp",), - "numerics.morphology": ("fast_dilation",), - "progress": ("log", "progress"), - "extraction.band.motion_binning": ( - "bin_motion", - "select_motion_bins", - "bin_motion_yx", - ), - "extraction.band.geometry": ( - "ref_pixs_to_drc", - "build_subsample_matrix_inds", - "_pad_to", - "build_combined_psf", - "threshold_and_crop_psf", - "build_selected_pixel_mask", - "pixel_coords_from_idxs", - "selected_pixels_2d_for_plane", - ), - "extraction.band.operators": ( - "build_sparse_h", - "expand_psf", - "shrink_psf", - "gaussian_kernel_2d", - "build_convolution_matrix", - "build_motion_h_matrices", - ), - "extraction.band.inputs": ( - "load_lookup_table", - "_ref_stack_group", - "find_reference_file", - "load_reference_stack", - "default_psf", - "load_psf", - ), - "extraction.band.background": ( - "_collect_interp_columns", - "_interp_columns", - "build_interp_data", - "baseline_window_frames", - "compute_rolling_baseline", - "assemble_background", - "fit_noise_variance_model", - "compute_residual", - ), - "extraction.band.activity": ( - "_plane_column_maps", - "_valid_sel_cols_for_motion", - "compute_rho", - "mask_high_nan_rho", - "decay_kernel_1d", - "smooth_rho", - ), - "extraction.band.localization": ( - "sel_pix_gaussian_profile", - "sel_pix_patch_profile", - "init_source_params", - "build_a_patches", - "build_x_support_mots", - "project_spatial_profiles", - "_motion_frames", - "solve_phi_motion", - "fit_phi_all_motions", - "multiplicative_nmf", - "fit_gaussian_profiles", - "variance_sortorder", - "reorder_sources", - "compute_source_snr", - "fit_sources", - ), - "extraction.band.summary_images": ( - "compute_mean_image", - "accumulate_activity_image", - "finalize_activity_image", - "phase_matched_nanmedian", - "snake_aligned_nanmedian", - ), - "extraction.band.traces": ( - "compute_high_res_traces", - "get_high_res_traces", - ), - "extraction.band.trial_data": ( - "compute_ds_frames", - "accumulate_superpixel_data", - "read_band_trial_data", - "assemble_lowres_data", - "save_lowres_data", - "load_lowres_data", - ), -} - - -def _source_tree(module): - """Parse a package module without importing its dependencies. - - Parameters - ---------- - module : str - Module path relative to ``giant_python``. - - Returns - ------- - ast.Module - Parsed source tree. - """ - path = PACKAGE_ROOT.joinpath(*module.split(".")).with_suffix(".py") - return ast.parse(path.read_text(encoding="utf-8-sig")) - - -def _imports(module, tree): - """Resolve static import targets, including function-local imports. - - Parameters - ---------- - module : str - Module path relative to ``giant_python``. - tree : ast.Module - Parsed source tree. - - Yields - ------ - str - Absolute imported module or member path. - """ - parent = module.rpartition(".")[0] - package = "giant_python" + ("." + parent if parent else "") - for node in ast.walk(tree): - if isinstance(node, ast.Import): - for alias in node.names: - yield alias.name - elif isinstance(node, ast.ImportFrom): - name = "." * node.level + (node.module or "") - target = importlib.util.resolve_name(name, package) - for alias in node.names: - yield target + "." + alias.name - - -class TestNumericArchitecture(unittest.TestCase): - """Guard canonical ownership and low-level import boundaries.""" - - def test_retired_namespaces_have_no_python_sources(self): - """Permit orphan caches, but never active modules or empty stubs.""" - for namespace in ("bandsilo", "math"): - with self.subTest(namespace=namespace): - self.assertEqual( - list((PACKAGE_ROOT / namespace).rglob("*.py")), [] - ) - - def test_production_imports_do_not_reference_retired_namespaces(self): - """Inspect imports, including relative/local ones, not docstrings.""" - retired = {"giant_python.bandsilo", "giant_python.math"} - for path in PACKAGE_ROOT.rglob("*.py"): - module = ".".join( - path.relative_to(PACKAGE_ROOT).with_suffix("").parts - ) - for target in _imports(module, _source_tree(module)): - with self.subTest(module=module, target=target): - self.assertFalse( - any( - target == name or target.startswith(name + ".") - for name in retired - ) - ) - - def test_public_exports_do_not_restore_retired_namespaces(self): - """Public exports cannot expose the removed compatibility APIs.""" - for name in ( - "giant_python", - "giant_python.numerics", - "giant_python.extraction", - "giant_python.extraction.band", - ): - namespace = importlib.import_module(name) - for retired in ("bandsilo", "math"): - with self.subTest(namespace=name, retired=retired): - self.assertNotIn( - retired, getattr(namespace, "__all__", ()) - ) - - def test_shared_numerics_do_not_import_application_layers(self): - """Shared kernels cannot depend on acquisition, IO, UI, or backends.""" - forbidden = {"h5py", "tifffile", "slap2_utils", "tkinter", "napari"} - for path in (PACKAGE_ROOT / "numerics").glob("*.py"): - module = "numerics." + path.stem - for target in _imports(module, _source_tree(module)): - with self.subTest(module=module, target=target): - self.assertNotIn(target.split(".")[0], forbidden) - self.assertTrue( - not target.startswith("giant_python.") - or target.startswith("giant_python.numerics.") - ) - - def test_band_kernels_do_not_depend_on_legacy_or_workflows(self): - """Keep workflows above kernels and acquisition at input boundaries.""" - blocked = ( - "giant_python.bandsilo.", - "giant_python.math.", - "giant_python.gui.", - "giant_python.pipeline.", - "giant_python.extraction.band.workflow.", - ) - input_boundaries = { - "extraction.band.inputs", - "extraction.band.trial_data", - "extraction.band.traces", - } - for module in OWNERS: - if not module.startswith("extraction.band."): - continue - for target in _imports(module, _source_tree(module)): - with self.subTest(module=module, target=target): - self.assertFalse(target.startswith(blocked)) - if module not in input_boundaries: - self.assertFalse(target.startswith("giant_python.io.")) - self.assertFalse( - target.startswith( - "giant_python.extraction.band.trial_data." - ) - ) - - def test_namespaces_do_not_eagerly_import_backends(self): - """Importing a namespace must not itself import IO or workflows.""" - for module in ( - "numerics.__init__", - "extraction.__init__", - "extraction.band.__init__", - ): - with self.subTest(module=module): - for node in _source_tree(module).body: - self.assertNotIsInstance( - node, (ast.Import, ast.ImportFrom) - ) - - def test_shared_helpers_and_summary_image_dependency(self): - """Acquisition-independent helpers live in numerics, not trial IO.""" - dependencies = { - "extraction.band.summary_images": ( - "giant_python.numerics.morphology.fast_dilation", - ), - "extraction.band.traces": ( - "giant_python.numerics.interpolation.nearest_interp", - ), - "extraction.band.trial_data": ( - "giant_python.io.slap2.open_slap2_file", - ), - } - for module, targets in dependencies.items(): - imports = set(_imports(module, _source_tree(module))) - for target in targets: - with self.subTest(module=module, target=target): - self.assertIn(target, imports) - - def test_kernels_have_one_actual_definition(self): - """Scan all production modules, not just the expected owner list.""" - definitions = {} - for path in PACKAGE_ROOT.rglob("*.py"): - module = ".".join( - path.relative_to(PACKAGE_ROOT).with_suffix("").parts - ) - for node in _source_tree(module).body: - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - definitions.setdefault(node.name, []).append(module) - for owner, names in OWNERS.items(): - for name in names: - with self.subTest(owner=owner, name=name): - self.assertEqual(definitions.get(name), [owner]) diff --git a/tests/test_numeric_edges.py b/tests/test_numeric_edges.py deleted file mode 100644 index 069ec10..0000000 --- a/tests/test_numeric_edges.py +++ /dev/null @@ -1,184 +0,0 @@ -"""Value-based regressions for previously uncovered numerical edge paths.""" - -import unittest -from unittest.mock import patch - -import numpy as np - -from giant_python.extraction.band import background, operators, summary_images -from giant_python.numerics import peaks - - -class TestOperatorEdges(unittest.TestCase): - """Protect normalization, centering, and nonnegative kernel weights.""" - - def test_identity_shrink_preserves_normalized_psf(self): - """Unit scales preserve a positive PSF up to sum normalization.""" - psf = np.arange(1, 26, dtype=np.float32).reshape(5, 5) - original = psf.copy() - actual = operators.shrink_psf(psf, scale_y=1, scale_x=1) - np.testing.assert_allclose(actual, psf / psf.sum(), rtol=1e-6) - np.testing.assert_array_equal(psf, original) - self.assertEqual(actual.dtype, np.float32) - - def test_shrink_narrows_only_requested_axis(self): - """An x-only shrink narrows the x moment without moving the center.""" - coords = np.arange(-6, 7, dtype=float) - psf = np.exp(-(coords[:, None] ** 2 + coords[None, :] ** 2) / 8) - actual = operators.shrink_psf(psf, scale_y=1, scale_x=0.5) - original = psf / psf.sum() - self.assertEqual(actual.shape, psf.shape) - self.assertTrue(np.all(actual >= 0)) - self.assertAlmostEqual(float(actual.sum()), 1, places=6) - np.testing.assert_allclose(actual, actual[::-1, ::-1], atol=1e-7) - self.assertLess( - np.sum(actual * coords[None, :] ** 2), - np.sum(original * coords[None, :] ** 2), - ) - np.testing.assert_allclose( - actual.sum(axis=1), original.sum(axis=1), atol=1e-6 - ) - - def test_nonpositive_psf_remains_zero_without_division(self): - """Clip negative weights and avoid normalizing zero total mass.""" - for value in (0, -1): - with self.subTest(value=value): - actual = operators.shrink_psf(np.full((5, 5), value)) - np.testing.assert_array_equal(actual, np.zeros((5, 5))) - self.assertEqual(actual.dtype, np.float32) - - def test_gaussian_kernel_matches_separable_weights(self): - """Use ceil-truncated support and a centered unit-mass Gaussian.""" - sigma, truncate = 0.8, 2.0 - kernel, center = operators.gaussian_kernel_2d(sigma, truncate) - self.assertEqual(center, (2, 2)) - actual = kernel.numpy() - coords = np.arange(-2, 3) - weights = np.exp(-(coords**2) / (2 * sigma**2)) - weights /= weights.sum() - np.testing.assert_allclose( - actual, np.outer(weights, weights), rtol=1e-6 - ) - self.assertEqual(actual.dtype, np.float32) - self.assertEqual( - np.unravel_index(actual.argmax(), actual.shape), center - ) - self.assertAlmostEqual(float(actual.sum()), 1, places=6) - - -class TestBackgroundEdges(unittest.TestCase): - """Check fallback medians, minimal windows, and interpolation rejection.""" - - def test_pandas_fallback_ignores_nan_and_preserves_dtype(self): - """Fallback clips odd windows and preserves empty-window NaNs.""" - data = np.array( - [[1, np.nan, 3, 100, 5], [np.nan] * 5], dtype=np.float32 - ) - original = data.copy() - with patch.object(background, "bn", None): - actual = background.compute_rolling_baseline(data, 3) - expected = np.array([[1, 2, 51.5, 5, 52.5], [np.nan] * 5]) - np.testing.assert_allclose(actual, expected, equal_nan=True) - np.testing.assert_array_equal(data, original) - self.assertEqual(actual.dtype, data.dtype) - - def test_single_frame_window_preserves_samples(self): - """Nonpositive and unit windows clamp to one without padding.""" - data = np.array([[1, np.nan, 7]], dtype=np.float32) - for window in (-2, 0, 1): - with self.subTest(window=window): - actual = background.compute_rolling_baseline(data, window) - np.testing.assert_array_equal(actual, data) - self.assertEqual(actual.dtype, data.dtype) - - def test_internal_interpolation_rejects_unknown_method(self): - """Reject invalid interpolation modes before mutating the output.""" - out = np.full((1, 1), np.nan, dtype=np.float32) - with self.assertRaisesRegex(ValueError, "linear.*cubic"): - background._interp_columns( - out, - {}, - {}, - {}, - np.array([0]), - np.ones((1, 1)), - np.array([0]), - np.float32, - method="nearest", - ) - self.assertTrue(np.isnan(out).all()) - - -class TestMedianGeometryEdges(unittest.TestCase): - """Reject malformed geometry instead of producing plausible medians.""" - - def test_phase_filter_rejects_invalid_geometry(self): - """Validate image, coordinate, depth, and selected-index contracts.""" - defaults = dict( - image=np.ones((1, 3, 3)), - valid_pix_idxs=np.arange(9), - ref_r=np.array([1]), - ref_c=np.array([1]), - ) - cases = [ - ({"image": np.ones((3, 3))}, "non-empty shape"), - ({"image": np.ones((1, 0, 3))}, "non-empty shape"), - ({"ref_r": []}, "non-empty and equally sized"), - ({"ref_c": [0, 1]}, "non-empty and equally sized"), - ({"ref_r": [np.nan]}, "reference rows must be finite"), - ({"ref_d": [0, 0]}, "same size"), - ({"ref_c": [0.5]}, "finite integers"), - ({"ref_d": [np.inf]}, "finite integers"), - ({"ref_d": [0.5]}, "finite integers"), - ({"valid_pix_idxs": [[0]]}, "one-dimensional integer array"), - ({"valid_pix_idxs": [0.5]}, "one-dimensional integer array"), - ({"valid_pix_idxs": [-1]}, "out-of-bounds"), - ({"valid_pix_idxs": [9]}, "out-of-bounds"), - ] - for overrides, message in cases: - with self.subTest(overrides=overrides): - with self.assertRaisesRegex(ValueError, message): - summary_images.phase_matched_nanmedian( - **(defaults | overrides) - ) - - def test_snake_filter_rejects_invalid_geometry(self): - """Require odd windows and at least one matched in-bounds center.""" - defaults = dict( - image=np.ones((1, 3, 3)), - valid_pix_idxs=np.arange(9), - ref_r=np.array([1]), - ref_c=np.array([1]), - ) - cases = [ - ({"height": 0}, "positive odd integers"), - ({"width": 2}, "positive odd integers"), - ({"ref_r": []}, "non-empty and equally sized"), - ({"ref_c": [0, 1]}, "non-empty and equally sized"), - ({"ref_c": [-1]}, "no reference centers"), - ({"ref_c": [3]}, "no reference centers"), - ] - for overrides, message in cases: - with self.subTest(overrides=overrides): - with self.assertRaisesRegex(ValueError, message): - summary_images.snake_aligned_nanmedian( - **(defaults | overrides) - ) - - -class TestPeakBoundsEdges(unittest.TestCase): - """Protect the shared-width initialization's feasible parameter domain.""" - - def test_disjoint_width_bounds_rejected_before_fitting(self): - """Disjoint y/x intervals cannot initialize an isotropic peak.""" - theta = np.array([[1, 0, 0, 0.5, 2.5]], dtype=float) - original = theta.copy() - lower = np.array([0, -1, -1, 0.25, 2]) - upper = np.array([2, 1, 1, 1, 3]) - with patch.object(peaks, "_lsq_curvefit") as fit: - with self.assertRaisesRegex(ValueError, "overlapping y/x"): - peaks._fit_isotropic_then_anisotropic( - theta, np.zeros((2, 1)), np.zeros(1), lower, upper - ) - fit.assert_not_called() - np.testing.assert_array_equal(theta, original) diff --git a/tests/test_package_metadata.py b/tests/test_package_metadata.py deleted file mode 100644 index 10da919..0000000 --- a/tests/test_package_metadata.py +++ /dev/null @@ -1,149 +0,0 @@ -"""Tests for package metadata declarations.""" - -import ast -import unittest -from importlib import import_module -from importlib.resources import files -from importlib.util import find_spec -from pathlib import Path - -from setuptools import find_packages - -SOURCE_ROOT = Path(__file__).resolve().parents[1] / "src" -PACKAGE_INIT = SOURCE_ROOT / "giant_python" / "__init__.py" - - -class PackageMetadataTest(unittest.TestCase): - """Tests for package metadata declarations.""" - - def test_canonical_packages_are_discoverable(self): - """Source discovery includes canonical owners, not retired packages.""" - packages = set(find_packages(where=str(SOURCE_ROOT))) - self.assertTrue( - { - "giant_python", - "giant_python.extraction", - "giant_python.extraction.band", - "giant_python.numerics", - }.issubset(packages) - ) - for package in packages: - with self.subTest(package=package): - self.assertFalse( - package == "giant_python.bandsilo" - or package.startswith("giant_python.bandsilo.") - or package == "giant_python.math" - or package.startswith("giant_python.math.") - ) - - def test_summary_modules_have_explicit_names(self): - """Summary owners keep public exports without old module shims.""" - import giant_python - - for owner, symbols in ( - ("models", ("ExperimentSummary", "PathSummary", "FrameInfo")), - ("io", ("read_summary", "write_summary")), - ): - package = import_module(f"giant_python.{owner}") - module = import_module(f"giant_python.{owner}.experiment_summary") - for name in symbols: - with self.subTest(owner=owner, symbol=name): - symbol = getattr(module, name) - self.assertEqual(symbol.__module__, module.__name__) - self.assertIs(getattr(package, name), symbol) - if owner == "models": - self.assertIs(getattr(giant_python, name), symbol) - self.assertIsNone(find_spec(f"giant_python.{owner}.experiment")) - old_path = SOURCE_ROOT / "giant_python" / owner / "experiment.py" - self.assertFalse(old_path.exists()) - - def test_band_and_gui_modules_have_explicit_names(self): - """Renamed modules own their functions without forwarding shims.""" - renames = { - "giant_python.extraction.band.results": ( - "result_assembly", - "assemble_path_summary", - ), - "giant_python.extraction.band.motion": ( - "motion_binning", - "bin_motion", - ), - "giant_python.gui.parameters": ( - "parameters_gui", - "run_parameter_gui", - ), - } - for old_name, (new_name, symbol) in renames.items(): - with self.subTest(module=new_name): - parent = old_name.rsplit(".", 1)[0] - module = import_module(f"{parent}.{new_name}") - self.assertEqual( - getattr(module, symbol).__module__, module.__name__ - ) - self.assertIsNone(find_spec(old_name)) - old_path = SOURCE_ROOT.joinpath( - *old_name.split(".") - ).with_suffix(".py") - self.assertFalse(old_path.exists()) - - def test_psf_assets_are_available_as_package_resources(self): - """Every bundled dilation template is readable without a build.""" - assets = files("giant_python").joinpath("assets", "psfs") - dilations = (*range(5, 32, 2), 41, 51) - for dilation in dilations: - name = f"dil-{dilation:02d}.tif" - with self.subTest(asset=name): - resource = assets.joinpath(name) - self.assertTrue(resource.is_file()) - with resource.open("rb") as stream: - self.assertIn( - stream.read(4), - ( - b"II\x2a\x00", - b"MM\x00\x2a", - b"II\x2b\x00", - b"MM\x00\x2b", - ), - ) - - def test_init_module_is_valid_and_has_single_version_line(self): - """Only one ``__version__`` line may exist so CI sed bumps stay safe. - - The shared AIND ``release-bump-version`` workflow rewrites every line - that contains ``__version__`` with:: - - sed 's|__version__.*|__version__ = "X.Y.Z"|' - - Listing ``"__version__"`` in ``__all__`` therefore produces invalid - Python on the next bump. Keep a single assignment line instead. - """ - source = PACKAGE_INIT.read_text(encoding="utf-8") - module = ast.parse(source) - - version_lines = [ - line for line in source.splitlines() if "__version__" in line - ] - self.assertEqual(len(version_lines), 1) - self.assertRegex( - version_lines[0].strip(), - r'^__version__ = "[0-9]+\.[0-9]+\.[0-9]+"$', - ) - - version_assignment = next( - node - for node in module.body - if isinstance(node, ast.Assign) - and any( - isinstance(target, ast.Name) and target.id == "__version__" - for target in node.targets - ) - ) - self.assertIsInstance(version_assignment.value, ast.Constant) - self.assertIsInstance(version_assignment.value.value, str) - self.assertRegex( - version_assignment.value.value, r"^[0-9]+\.[0-9]+\.[0-9]+$" - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_parallel.py b/tests/test_parallel.py deleted file mode 100644 index e9be201..0000000 --- a/tests/test_parallel.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Tests for the per-trial parallelism helper.""" - -import unittest - -from giant_python.parallel import map_trials - - -def _double(x): - """Module-level (picklable) worker for the parallel path.""" - return x * 2 - - -class TestMapTrials(unittest.TestCase): - """Tests for the per-trial parallelism helper.""" - - def test_serial(self): - """n_workers=1 runs serially, preserving order.""" - self.assertEqual( - map_trials(_double, [1, 2, 3], n_workers=1), [2, 4, 6] - ) - - def test_single_trial_runs_serial(self): - """A single trial short-circuits to the serial path.""" - self.assertEqual(map_trials(_double, [5], n_workers=4), [10]) - - def test_parallel(self): - """Multiple workers fan out and return results in input order.""" - self.assertEqual( - map_trials(_double, [1, 2, 3, 4], n_workers=2), [2, 4, 6, 8] - ) - - def test_desc_shows_progress_bar(self): - """A desc label wraps the trials in a bar without changing results.""" - self.assertEqual( - map_trials(_double, [1, 2, 3], desc="doubling"), [2, 4, 6] - ) - - def test_parallel_progress_preserves_order(self): - """Progress wraps completed worker results without changing order.""" - self.assertEqual( - map_trials(_double, [3, 1, 2], n_workers=2, desc="parallel"), - [6, 2, 4], - ) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_parameter_gui.py b/tests/test_parameter_gui.py deleted file mode 100644 index d52c131..0000000 --- a/tests/test_parameter_gui.py +++ /dev/null @@ -1,166 +0,0 @@ -"""Headless checks for the parameter form's canonical split-option API.""" - -import math -import sys -import unittest -from unittest.mock import MagicMock, patch - -from giant_python.gui.parameters_gui import run_parameter_gui -from giant_python.models.params import ( - AnnotationOptions, - BandSiloParams, - ExecutionOptions, -) - - -class TestParameterGui(unittest.TestCase): - """Exercise form callbacks without creating a desktop window.""" - - def setUp(self): - """Replace toolkit widgets with controlled variables and callbacks.""" - self.tk = MagicMock() - self.root = self.tk.Tk.return_value - self.variables = [] - self.boolean_variables = [] - self.buttons = {} - self.tk.StringVar.side_effect = self._string_var - self.tk.BooleanVar.side_effect = self._boolean_var - self.tk.ttk.Button.side_effect = self._button - self.modules = patch.dict(sys.modules, {"tkinter": self.tk}) - self.modules.start() - self.addCleanup(self.modules.stop) - - def _string_var(self, *, value): - """Retain each field variable so simulated users can edit it.""" - variable = MagicMock() - variable.get.return_value = value - self.variables.append(variable) - return variable - - def _boolean_var(self, *, value): - """Track checkbox controls separately from text fields.""" - self.roi_var = MagicMock() - self.roi_var.get.return_value = value - self.boolean_variables.append(self.roi_var) - return self.roi_var - - def _button(self, parent, *, text, command): - """Capture form actions instead of binding real toolkit events.""" - self.buttons[text] = command - return MagicMock() - - def test_success_returns_fresh_split_options(self): - """Scientific, execution, and annotation controls stay independent.""" - params = BandSiloParams(peakth=8) - run = ExecutionOptions(max_workers=2, verbose=True, max_trials=4) - roi = AnnotationOptions(interactive=False) - - def interact(): - """Simulate edits to a scientific field and both policy groups.""" - self.variables[0].get.return_value = "125" - self.variables[8].get.return_value = "3" - self.variables[9].get.return_value = "2.5" - self.variables[10].get.return_value = "-2" - self.boolean_variables[0].get.return_value = True - self.roi_var.get.return_value = True - self.buttons["OK"]() - - self.root.mainloop.side_effect = interact - result = run_parameter_gui(params, execution=run, annotations=roi) - assert result is not None - science, execution, annotations = result - self.assertEqual(science.analyze_hz, 125) - self.assertEqual(science.peakth, 8) - self.assertEqual(science.z_tol, 2.5) - for name in ("simple_trace_extraction", "convolution_trace_extraction"): - self.assertFalse(hasattr(science, name)) - self.assertFalse(hasattr(params, name)) - self.assertTrue(science.trace_debias) - self.assertFalse(params.trace_debias) - self.assertEqual(params.z_tol, 1.5) - self.assertAlmostEqual(science.sparse_fac, math.exp(-2)) - labels = [ - call.kwargs["text"] for call in self.tk.ttk.Label.call_args_list - ] - self.assertNotIn("Operator:", labels) - self.assertEqual(execution, ExecutionOptions(3, True, 4)) - self.assertEqual(annotations, AnnotationOptions(True, False)) - self.assertEqual(params.analyze_hz, 100) - self.assertEqual(run.max_workers, 2) - self.assertFalse(roi.enabled) - self.assertFalse(roi.interactive) - for supplied, returned in zip((params, run, roi), result): - self.assertIsNot(supplied, returned) - self.root.destroy.assert_called_once() - self.tk.messagebox.showerror.assert_not_called() - - def test_cancel_returns_none(self): - """Cancel does not manufacture a combined parameter result.""" - self.root.mainloop.side_effect = lambda: self.buttons["Cancel"]() - self.assertIsNone(run_parameter_gui()) - self.root.destroy.assert_called_once() - - def test_removed_trace_flags_are_rejected_before_opening_form(self): - """Removed dictionary keys cannot enter the GUI's base configuration.""" - for name in ("simple_trace_extraction", "convolution_trace_extraction"): - for value in (False, True): - with self.subTest(name=name, value=value): - supplied = {name: value} - with self.assertRaisesRegex(TypeError, name): - run_parameter_gui(supplied) - self.assertEqual(supplied, {name: value}) - self.tk.Tk.assert_not_called() - self.root.mainloop.assert_not_called() - - def test_convolution_controls_preserve_trace_settings(self): - """Accepting the GUI retains the independent trace-only options.""" - params = BandSiloParams( - decay_tau_s=.3, - trace_beta_fraction=.02, trace_max_iter=900, - trace_tolerance=1e-9, trace_debias=True, trace_event_threshold=.001, - ) - self.root.mainloop.side_effect = lambda: self.buttons["OK"]() - result = run_parameter_gui(params) - self.assertEqual(result[0], params) - self.assertIsNot(result[0], params) - self.assertEqual(result[0].decay_tau_s, .3) - - def test_trace_mode_toggles_are_replaced_by_always_saved_label(self): - """Only debias and ROI checkboxes remain; extraction is not optional.""" - self.root.mainloop.side_effect = lambda: self.buttons["OK"]() - result = run_parameter_gui() - labels = [call.kwargs["text"] for call in self.tk.ttk.Label.call_args_list] - self.assertIn("LS and convolution traces are always saved.", labels) - self.assertEqual(len(self.boolean_variables), 2) - self.assertEqual(self.tk.ttk.Checkbutton.call_count, 2) - self.assertEqual( - [label for label in labels if label.endswith("?")], - ["Debias Convolution Traces?", "Draw User ROIs?"], - ) - for call, variable in zip( - self.tk.ttk.Checkbutton.call_args_list, self.boolean_variables - ): - self.assertIs(call.kwargs["variable"], variable) - for name in ("simple_trace_extraction", "convolution_trace_extraction"): - self.assertFalse(hasattr(result[0], name)) - self.assertFalse(result[2].enabled) - self.tk.messagebox.showerror.assert_not_called() - - def test_invalid_input_shows_error_without_accepting(self): - """Invalid scientific values leave the form open for correction.""" - - def interact(): - """Try a nonpositive rate, then cancel the still-open form.""" - self.variables[0].get.return_value = "0" - self.buttons["OK"]() - self.root.destroy.assert_not_called() - self.tk.messagebox.showerror.assert_called_once() - self.buttons["Cancel"]() - - self.root.mainloop.side_effect = interact - self.assertIsNone(run_parameter_gui()) - self.root.destroy.assert_called_once() - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_path_summary.py b/tests/test_path_summary.py deleted file mode 100644 index 8afa4fb..0000000 --- a/tests/test_path_summary.py +++ /dev/null @@ -1,420 +0,0 @@ -"""Nonzero two-path assembly, model, and emitted-schema round-trip coverage.""" - -import tempfile -import unittest -from pathlib import Path - -import h5py -import numpy as np - -from giant_python.extraction.band.inputs import load_alignment_data_h5 -from giant_python.extraction.band.result_assembly import ( - PathResult, - assemble_path_summary, -) -from giant_python.extraction.band.types import TrialTraceResult -from giant_python.io.experiment_summary import ( - read_summary, - write_summary, -) -from giant_python.models.experiment_summary import ( - FRAME_INFO_KEYS, - ExperimentSummary, - FrameInfo, -) -from tests.test_bandsilo_pipeline import _path_result - - -class _ArrayTensor: - """Minimal tensor interface for assembly without a torch import.""" - - def __init__(self, array): - """Keep the supplied array unchanged.""" - self.array = array - - def numpy(self): - """Expose the backing array, like the CPU tensor producer.""" - return self.array - - def __getitem__(self, key): - """Return a tensor-like slice for source-parameter indexing.""" - return _ArrayTensor(self.array[key]) - - -def _write_alignment(path, index): - """Write signed fractional displacement knots with a missing sample.""" - saved = { - "motionDSr": np.array([1.25, -2.5, np.nan]) + index, - "motionDSc": np.array([-3.5, 0.75, 2.25]) - index, - "motionDSz": np.array([0.5, -1.25, 3.0]) + 2 * index, - } - with h5py.File(path, "w") as f: - f["row_major"] = 1 - f["DSframes"] = np.array([[1, 5, 9]]) - f["numChannels"] = 2 - f["alignHz"] = 10.0 - for key, value in saved.items(): - f[key] = value.reshape(-1, 1) - online = f.create_group("slap2") - for axis in ("X", "Y", "Z"): - online[f"onlineMotion{axis}shift"] = np.array([[2, -3, 4]]) - return saved - - -def _result(index, alignment, roi_state="present"): - """Build two stitched trials with distinct path shapes and ROI policies.""" - n_sources, channels, nz, rows, cols = ( - (2, 2, 1, 3, 4) if index == 0 else (3, 1, 2, 2, 3) - ) - frames = np.arange(1, 10, dtype=float) - # Match the existing band linear/clamped interpolation, not MATLAB PCHIP. - offline = tuple( - np.interp(frames, alignment["DSframes"], alignment[key]) - for key in ("motionDSr", "motionDSc", "motionDSz") - ) - online = tuple(np.arange(9, dtype=np.int16) + axis for axis in (3, -2, 1)) - n_rois = 1 if roi_state == "present" else 0 - traces = [] - for section in (slice(0, 4), slice(4, 9)): - n_frames = frames[section].size - signal = ( - np.arange(n_frames * n_sources, dtype=np.float32).reshape( - n_frames, n_sources - ) - / 10 - ) - signal[1] = np.nan - denoised = signal + 100 * (index + 1) + section.start - events = np.zeros_like(signal) - events[-1] = np.arange(1, n_sources + 1) + section.start - events[1] = np.nan - traces.append( - TrialTraceResult( - signal, - np.full_like(signal, 5.0), - frames[section], - np.arange(3), - np.full((n_frames, channels), index + 2.0, dtype=np.float32), - tuple(axis[section] for axis in offline), - tuple(axis[section] for axis in online), - np.full( - (n_frames, n_rois, channels), index + 7.0, dtype=np.float32 - ), - denoised, - events, - ) - ) - masks = [np.ones((nz, rows, cols), dtype=bool)] if n_rois else [] - return PathResult( - a=_ArrayTensor( - np.arange(nz * rows * cols * n_sources, dtype=np.float32).reshape( - -1, n_sources - ) - ), - source_params=_ArrayTensor( - np.arange(n_sources * 6, dtype=np.float32).reshape(n_sources, 6) - ), - source_snr=np.arange(1, n_sources + 1, dtype=np.float32), - act_im=np.full((nz, rows, cols), index + 0.25, dtype=np.float32), - mean_im=np.full( - (channels, nz, rows, cols), index + 4.5, dtype=np.float32 - ), - act_im_peaks=np.zeros((n_sources, 3), dtype=np.int32), - trace_results=traces, - z_depths=np.arange(nz).reshape(-1, 1), - num_fast_zs=nz, - dmd_pixels_per_column=rows, - dmd_pixels_per_row=cols, - num_channels=channels, - denoise_window=3, - baseline_window=5, - draw_user_rois=roi_state != "absent", - soma_masks=masks if roi_state != "absent" else None, - soma_labels=["ROI μ"] if n_rois else [], - yx_shape=(rows, cols), - ) - - -def _datasets(path): - """Return raw dataset values and dtypes for complete schema comparison.""" - values = {} - with h5py.File(path, "r") as f: - - def collect(name, item): - """Collect datasets, retaining stored shape/dtype and values.""" - if isinstance(item, h5py.Dataset): - values[name] = (item[()], item.dtype) - - f.visititems(collect) - return values - - -class TestPathSummaryRoundTrip(unittest.TestCase): - """Per-path data and corrected public shifts survive HDF5 round trips.""" - - def test_nonzero_two_path_model_and_schema_roundtrip(self): - """Raw -> normalized -> interpolated -> assembled -> model -> disk.""" - with tempfile.TemporaryDirectory() as dr: - results, paths, expected = [], [], [] - for index in range(2): - alignment_path = Path(dr) / f"alignment{index}.h5" - saved = _write_alignment(alignment_path, index) - result = _result( - index, - load_alignment_data_h5(alignment_path), - "present" if index == 0 else "empty", - ) - results.append(result) - paths.append(assemble_path_summary(result, index)) - expected.append( - { - axis: np.interp( - np.arange(1, 10), [1, 5, 9], saved[key] - ) - for axis, key in ( - ("Y", "motionDSr"), - ("X", "motionDSc"), - ("Z", "motionDSz"), - ) - } - ) - summary = ExperimentSummary( - params={ - "operator": "μ", - "numChannels": 2, - "draw_user_rois": True, - }, - paths=paths, - ) - self.assertEqual([p.path_index for p in summary.paths], [0, 1]) - self.assertEqual([len(p.sources) for p in summary.paths], [2, 3]) - self.assertEqual([len(p.user_rois) for p in summary.paths], [1, 0]) - self.assertTrue(all(p.annotation_enabled for p in summary.paths)) - for index, path in enumerate(summary.paths): - self.assertIsInstance(path.frame_info, FrameInfo) - expected_denoised = np.concatenate([ - r.d_f_denoised for r in results[index].trace_results - ]) - expected_events = np.concatenate([ - r.events for r in results[index].trace_results - ]) - for source_index, source in enumerate(path.sources): - for field, expected_trace in ( - ("df_denoised", expected_denoised), - ("events", expected_events), - ): - values = getattr(source, field) - self.assertEqual(values.shape, (results[index].num_channels, 9)) - np.testing.assert_array_equal(values[0], expected_trace[:, source_index]) - self.assertTrue(np.isnan(values[1:]).all()) - self.assertFalse(np.allclose( - source.df_ls, source.df_denoised, equal_nan=True - )) - self.assertIs( - path.frame_info.to_dict()["offlineXshifts"], - path.frame_info.offlineXshifts, - ) - self.assertTrue( - np.shares_memory( - path.sources[0].profile, results[index].a.numpy() - ) - ) - for axis in ("X", "Y", "Z"): - np.testing.assert_allclose( - getattr( - path.frame_info, f"offline{axis}shifts" - ).ravel(), - expected[index][axis], - equal_nan=True, - ) - for axis, position in (("X", 1), ("Y", 0), ("Z", 2)): - np.testing.assert_array_equal( - getattr( - path.frame_info, f"online{axis}shifts" - ).ravel(), - np.concatenate( - [ - r.online_shifts[position] - for r in results[index].trace_results - ] - ), - ) - # Assembling output did not invert the private input arrays. - np.testing.assert_allclose( - -np.concatenate( - [ - r.reference_offsets[1] - for r in results[index].trace_results - ] - ), - expected[index]["X"], - equal_nan=True, - ) - model_file = Path(dr) / "model.h5" - roundtrip_file = Path(dr) / "roundtrip.h5" - self.assertIsNone(write_summary(summary, model_file)) - reloaded = ExperimentSummary.from_h5(model_file) - self.assertIsNone(reloaded.to_h5(roundtrip_file)) - original = _datasets(model_file) - for filename in (roundtrip_file,): - actual = _datasets(filename) - self.assertEqual(set(actual), set(original)) - for key, (value, dtype) in original.items(): - with self.subTest(file=filename.name, dataset=key): - self.assertEqual(actual[key][1], dtype) - np.testing.assert_array_equal(actual[key][0], value) - for index, path in enumerate(reloaded.paths): - self.assertEqual(path.path_index, index) - for before, after in zip(paths[index].sources, path.sources): - for field in ("df_ls", "f0", "df_denoised", "events"): - np.testing.assert_array_equal( - getattr(after, field), getattr(before, field) - ) - np.testing.assert_array_equal( - path.visualizations.mean_im, results[index].mean_im - ) - np.testing.assert_array_equal( - path.global_f, paths[index].global_f - ) - np.testing.assert_array_equal( - path.z_depths, results[index].z_depths - ) - with h5py.File(model_file, "r") as f: - self.assertEqual(f["Path2/user_rois/mask"].shape, (0, 2, 2, 3)) - self.assertEqual(f["Path2/user_rois/F"].shape, (0, 1, 9)) - self.assertEqual(f["Path2/user_rois/Fsvd"].shape, (0, 1, 9)) - self.assertEqual(f["Path2/user_rois/labels"].shape, (0, 1)) - self.assertTrue(f["Path2/user_rois"].attrs["annotation_enabled"]) - self.assertTrue(np.isnan(f["Path1/user_rois/Fsvd"][()]).all()) - for index, path in enumerate(paths): - for key, field in (("dF_denoised", "df_denoised"), - ("events", "events")): - np.testing.assert_array_equal( - f[f"Path{index + 1}/sources/temporal/{key}"][()], - np.stack([getattr(s, field) for s in path.sources]), - ) - for key in FRAME_INFO_KEYS: - self.assertEqual(f[f"Path1/frame_info/{key}"].ndim, 2) - - def test_explicit_paths_and_typed_frame_info_only(self): - """No aggregate, single-path convenience or mapping APIs remain.""" - data = { - "DSframes": [1, 5, 9], - **{ - k: np.zeros(3) for k in ("motionDSr", "motionDSc", "motionDSz") - }, - } - first = assemble_path_summary(_result(0, data), path_index=0) - second = assemble_path_summary(_result(1, data), path_index=1) - summary = ExperimentSummary(paths=[first, second]) - for model in ( - summary, - ExperimentSummary(paths=[first]), - ExperimentSummary(), - ): - for name in ( - "frame_info", - "visualizations", - "z_depths", - "global_f", - "sources", - "user_rois", - "_single_path", - ): - with self.subTest(paths=len(model.paths), name=name): - self.assertFalse(hasattr(model, name)) - self.assertIs(summary.paths[1].sources, second.sources) - self.assertIs(summary.paths[0].user_rois, first.user_rois) - for name in ("__getitem__", "__iter__", "__len__"): - self.assertFalse(hasattr(first.frame_info, name)) - self.assertEqual(set(first.frame_info.to_dict()), set(FRAME_INFO_KEYS)) - - def test_absent_rois_and_overwrite_semantics(self): - """Disabled differs from enabled-empty by attribute, not group absence.""" - data = { - "DSframes": [1, 5, 9], - **{ - k: np.zeros(3) for k in ("motionDSr", "motionDSc", "motionDSz") - }, - } - first = assemble_path_summary(_result(0, data, "empty"), 0) - second = assemble_path_summary(_result(1, data, "present"), 1) - with tempfile.TemporaryDirectory() as dr: - path = Path(dr) / "summary.h5" - write_summary( - ExperimentSummary( - paths=[first, second], params={"operator": "original"} - ), - path, - ) - replacement = assemble_path_summary(_result(0, data, "absent"), 0) - write_summary( - ExperimentSummary( - paths=[replacement], params={"operator": "new"} - ), - path, - ) - loaded = read_summary(path) - self.assertFalse(loaded.paths[0].annotation_enabled) - self.assertTrue(loaded.paths[1].annotation_enabled) - self.assertEqual(loaded.params["operator"], "original") - with h5py.File(path, "r") as f: - rois = f["Path1/user_rois"] - self.assertFalse(rois.attrs["annotation_enabled"]) - self.assertEqual(set(rois), {"mask", "labels", "F", "Fsvd"}) - self.assertEqual(rois["mask"].shape, (0, 1, 3, 4)) - self.assertEqual(rois["labels"].shape, (0, 1)) - self.assertEqual(rois["F"].shape, (0, 2, 9)) - self.assertEqual(rois["Fsvd"].shape, (0, 2, 9)) - self.assertTrue(f["Path2/user_rois"].attrs["annotation_enabled"]) - - def test_empty_sources_retain_shape_and_dtype(self): - """Numerical assembly and codecs retain empty tensor layout.""" - empty = assemble_path_summary(_path_result(n_sources=0), 2) - summary = ExperimentSummary(paths=[empty]) - with tempfile.TemporaryDirectory() as dr: - path = Path(dr) / "empty.h5" - summary.to_h5(path) - restored = read_summary(path) - self.assertEqual(restored.paths[0].path_index, 2) - self.assertEqual(restored.paths[0].sources, []) - for key, arr in restored.paths[0]._empty_source_arrays.items(): - expected = empty._empty_source_arrays[key] - self.assertEqual(arr.shape, expected.shape) - self.assertEqual(arr.dtype, expected.dtype) - - def test_column_major_summary_decodes_without_sign_conversion(self): - """The summary codec uses the same 3D/4D axis convention as raw IO.""" - data = { - "DSframes": [1, 5, 9], - "motionDSr": np.array([1.25, -3.5, 2.0]), - "motionDSc": np.array([-2.5, 0.25, 4.0]), - "motionDSz": np.array([0.5, 1.5, -0.75]), - } - summary = ExperimentSummary( - paths=[ - assemble_path_summary(_result(0, data, "present"), 0), - assemble_path_summary(_result(1, data, "empty"), 1), - ] - ) - with tempfile.TemporaryDirectory() as dr: - original = Path(dr) / "row.h5" - matlab = Path(dr) / "column.h5" - restored = Path(dr) / "restored.h5" - write_summary(summary, original) - datasets = _datasets(original) - with h5py.File(matlab, "w") as f: - for name, (value, dtype) in datasets.items(): - array = np.asarray(value) - if name == "row_major": - array = np.asarray(0) - elif array.ndim >= 2: - array = array.T - f.create_dataset(name, data=array, dtype=dtype) - loaded = read_summary(matlab) - write_summary(loaded, restored) - rewritten = _datasets(restored) - for name, (value, dtype) in datasets.items(): - with self.subTest(dataset=name): - self.assertEqual(rewritten[name][1], dtype) - np.testing.assert_array_equal(rewritten[name][0], value) diff --git a/tests/test_pipeline.py b/tests/test_pipeline.py deleted file mode 100644 index 0b9cf0d..0000000 --- a/tests/test_pipeline.py +++ /dev/null @@ -1,82 +0,0 @@ -"""Tests for the implemented annotation and extraction pipeline facade.""" - -import unittest -from pathlib import Path -from unittest import mock - -from giant_python.models import BandSiloParams, ExperimentSummary, TrialTable -from giant_python.pipeline import Pipeline - - -class TestPipelineFacade(unittest.TestCase): - """Tests for the Pipeline facade.""" - - def setUp(self): - """Construct a pipeline for each test.""" - self.pipe = Pipeline(microscope="slap2") - - def test_holds_session_state(self): - """The facade holds config and starts with no data.""" - self.assertEqual(self.pipe.microscope, "slap2") - self.assertIsNone(self.pipe.tt) - self.assertIsNone(self.pipe.summary) - self.assertIsInstance(self.pipe.params, BandSiloParams) - self.assertEqual(self.pipe.scan_mode, "band") - self.assertFalse(hasattr(self.pipe, "silo_params")) - - def test_params_keyword(self): - """Only the canonical scientific keyword is accepted.""" - params = BandSiloParams() - self.assertIs(Pipeline(params=params).params, params) - with self.assertRaises(TypeError): - Pipeline(silo_params=params) - - def test_annotate_delegates(self): - """Explicit standard annotation fails instead of pretending to work.""" - self.pipe.scan_mode = "standard" - self.pipe.tt = TrialTable() - with self.assertRaisesRegex(ValueError, "standard"): - self.pipe.annotate() - - def test_annotate_band_delegates(self): - """A band scan mode uses the BandSILo standalone annotation step.""" - self.pipe.tt = TrialTable(savedr=Path("/results")) - with mock.patch( - "giant_python.pipeline.pipeline.annotate_rois", - return_value="annotations.h5", - ) as annotate_band: - result = self.pipe.annotate() - annotate_band.assert_called_once() - self.assertIs(annotate_band.call_args[0][0], self.pipe.tt) - self.assertTrue(self.pipe.annotations.enabled) - self.assertIs(result, self.pipe) - - def test_extract_delegates(self): - """extract dispatches by scan mode and stores the summary.""" - self.pipe.tt = TrialTable() - summary = ExperimentSummary() - with mock.patch( - "giant_python.pipeline.pipeline.extract_sources", - return_value=summary, - ) as extractor: - result = self.pipe.extract() - self.assertIs(result, self.pipe) - self.assertIs(self.pipe.summary, summary) - self.assertIs(extractor.call_args[0][0], self.pipe.tt) - - def test_from_trial_table(self): - """from_trial_table loads once and retains the provided filename.""" - table = TrialTable(savedr=Path("/results")) - path = Path("/data/nonstandard_name.h5") - with mock.patch.object( - TrialTable, "from_h5", return_value=table - ) as load: - pipeline = Pipeline.from_trial_table(path) - load.assert_called_once_with(path) - self.assertIs(pipeline.tt, table) - self.assertEqual(pipeline.trial_table_path, path) - self.assertEqual(table.source_path, path) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_staging_example.py b/tests/test_staging_example.py deleted file mode 100644 index 50c8465..0000000 --- a/tests/test_staging_example.py +++ /dev/null @@ -1,559 +0,0 @@ -"""Safety and relocation contracts for the standalone staging example.""" - -import contextlib -import io -import os -import subprocess -import sys -import tempfile -import textwrap -import unittest -from dataclasses import asdict -from pathlib import Path -from types import SimpleNamespace -from unittest.mock import call, patch - -from examples import extract_band as example -from giant_python import AnnotationOptions, BandSiloParams, ExecutionOptions -from giant_python.models.trial_table import Slap2Info, TrialTable - -ROOT = Path(__file__).resolve().parents[1] - - -class TestExampleImport(unittest.TestCase): - """Import and help work without loading the scientific backend.""" - - def test_import_has_no_filesystem_environment_or_backend_side_effects( - self, - ): - """Execute an import in a fresh interpreter with mutation guards.""" - code = textwrap.dedent(""" - import argparse - import builtins - import contextlib - import dataclasses - import os - import pathlib - import shutil - import sys - import types - import typing - from unittest.mock import patch - - path = pathlib.Path('examples/extract_band.py') - source = compile(path.read_text(), str(path), 'exec') - module = types.ModuleType('staging_import_check') - sys.modules[module.__name__] = module - before = dict(os.environ) - original_import = builtins.__import__ - forbidden = { - 'giant_python', 'numpy', 'scipy', 'matplotlib', 'cv2', - 'tkinter', 'joblib', 'slap2_utils', 'torch', 'h5py', - } - - def checked_import(name, *args, **kwargs): - assert name.split('.')[0] not in forbidden, name - return original_import(name, *args, **kwargs) - - def audit(event, args): - assert event not in { - 'open', 'os.mkdir', 'os.remove', 'os.rmdir', 'os.rename', - 'os.symlink', 'os.link', 'os.putenv', 'os.unsetenv', - 'os.chdir', 'os.chmod', 'os.truncate', 'os.utime', - 'os.listdir', 'os.scandir', - }, (event, args) - - sys.addaudithook(audit) - with patch('builtins.__import__', side_effect=checked_import): - exec(source, module.__dict__) - assert dict(os.environ) == before - assert not forbidden.intersection(sys.modules) - assert callable(module.main) - """) - result = subprocess.run( - [sys.executable, "-B", "-c", code], - cwd=ROOT, - capture_output=True, - text=True, - timeout=30, - ) - self.assertEqual(result.returncode, 0, result.stdout + result.stderr) - - def test_help_exits_before_validation(self): - """Help requires neither a dataset nor GIAnT imports.""" - with ( - patch.object(example, "prepare_run") as prepare, - contextlib.redirect_stdout(io.StringIO()) as output, - self.assertRaises(SystemExit) as raised, - ): - example.main(["--help"]) - self.assertEqual(raised.exception.code, 0) - self.assertIn("--stage-data", output.getvalue()) - self.assertNotIn("--operator", output.getvalue()) - prepare.assert_not_called() - - -class TestRelativeNames(unittest.TestCase): - """Metadata cannot bypass input roots on either path convention.""" - - def test_relative_nested_and_empty_entries_are_accepted(self): - """Preserve relative subdirectories and absent per-trial entries.""" - example.validate_relative_names( - [ - [None, "", "trial.dat"], - ["raw/trial.dat", r"raw\trial.dat", "."], - ], - "filename", - ) - - def test_absolute_drive_unc_and_parent_names_are_rejected(self): - """POSIX and Windows escapes fail regardless of the host OS.""" - for name in ( - "/raw/trial.dat", - "../trial.dat", - "raw/../trial.dat", - r"..\trial.dat", - r"raw\..\trial.dat", - r"raw/..\trial.dat", - r"C:\raw\trial.dat", - "C:/raw/trial.dat", - "C:trial.dat", - r"\raw\trial.dat", - r"\\server\share\trial.dat", - "//server/share/trial.dat", - ): - for field in ("filename", "motion_correction/fn_adata"): - with self.subTest(name=name, field=field): - with self.assertRaisesRegex(ValueError, field): - example.validate_relative_names([["ok", name]], field) - - -class TestStagingExample(unittest.TestCase): - """Use tiny fixture files and a mocked backend, never a real dataset.""" - - def setUp(self): - """Create independent input trees and a portable loaded table.""" - temporary = tempfile.TemporaryDirectory(prefix="giant-example-") - self.addCleanup(temporary.cleanup) - self.root = Path(temporary.name).resolve() - self.data = self.root / "raw" - self.motion = self.root / "registered" - self.annotations = self.root / "rois" - self.scratch = self.root / "scratch" - self.results = self.root / "run" - self.table_path = self.root / "selected-table.h5" - for directory in ( - self.data, - self.motion, - self.annotations, - self.scratch, - ): - directory.mkdir() - self.table_path.write_bytes(b"unchanged table provenance") - (self.data / "trial.dat").write_bytes(b"raw") - (self.data / "trial.meta").write_bytes(b"sidecar") - (self.data / "reference.tif").write_bytes(b"reference") - (self.motion / "bandRegLookupTable.h5").write_bytes(b"lookup") - (self.motion / "alignment.h5").write_bytes(b"alignment") - (self.annotations / "annotations.h5").write_bytes(b"annotations") - self.table = TrialTable( - datadr=Path("old/raw"), - savedr=Path("old/results"), - filename=[["trial.dat"]], - slap2_info=Slap2Info(), - motion_correction={"fn_adata": [["alignment.h5"]]}, - source_path=Path("old/table.h5"), - ) - self.load = self.enterContext( - patch.object(TrialTable, "from_h5", return_value=self.table) - ) - self.extract = self.enterContext( - patch( - "giant_python.extract_band_sources", - return_value=SimpleNamespace(paths=[object()]), - ) - ) - - def argv(self, *extra): - """Build explicit input arguments, allowing final flag overrides.""" - return [ - "--trial-table", - str(self.table_path), - "--data-dir", - str(self.data), - "--motion-correction", - str(self.motion), - "--results-dir", - str(self.results), - *map(str, extra), - ] - - def invoke(self, *extra): - """Run quietly with the mocked extraction service.""" - with contextlib.redirect_stdout(io.StringIO()): - return example.main(self.argv(*extra)) - - def snapshot(self): - """Capture fixture contents, including links without following them.""" - return { - str(path.relative_to(self.root)): ( - ("link", str(path.readlink())) - if path.is_symlink() - else ( - ("directory", None) - if path.is_dir() - else ("file", path.read_bytes()) - ) - ) - for path in self.root.rglob("*") - } - - def assert_rejected(self, error, *extra): - """Rejected inputs must not stage, extract, write or change env.""" - before = self.snapshot() - environment = dict(os.environ) - extraction_calls = list(self.extract.mock_calls) - with ( - patch.object(example, "stage_directory") as stage, - patch.object(example.shutil, "copy2") as copy, - patch.object(Path, "mkdir") as mkdir, - contextlib.redirect_stderr(io.StringIO()), - self.assertRaises(error), - ): - self.invoke("--joblib-temp", self.scratch, *extra) - stage.assert_not_called() - copy.assert_not_called() - mkdir.assert_not_called() - self.assertEqual(self.extract.mock_calls, extraction_calls) - self.assertEqual(self.snapshot(), before) - self.assertEqual(dict(os.environ), environment) - - def test_default_run_relocates_loaded_model_without_rewriting_source(self): - """Default staging changes only the copied model's effective roots.""" - original = asdict(self.table) - self.assertEqual(self.invoke(), 0) - self.load.assert_called_once_with(self.table_path) - self.extract.assert_called_once() - args, kwargs = self.extract.call_args - table, science = args - self.assertIsInstance(table, TrialTable) - self.assertIsNot(table, self.table) - self.assertEqual(table.datadr, self.data) - self.assertEqual(table.savedr, self.results) - self.assertEqual( - table.source_path, self.results / "input_trial_table.h5" - ) - self.assertEqual(asdict(self.table), original) - self.assertEqual(science, BandSiloParams()) - self.assertEqual(science.peakth, 7.0) - self.assertEqual(kwargs["execution"], ExecutionOptions()) - self.assertEqual( - kwargs["annotations"], - AnnotationOptions(False, False), - ) - self.assertEqual( - table.source_path.read_bytes(), self.table_path.read_bytes() - ) - self.assertEqual( - (self.results / "motion_correction/alignment.h5").read_bytes(), - b"alignment", - ) - self.assertFalse((self.results / "input").exists()) - - def test_operator_option_is_rejected_before_staging(self): - """The removed CLI option fails before any extraction or staging.""" - before = self.snapshot() - with ( - contextlib.redirect_stderr(io.StringIO()) as stderr, - self.assertRaises(SystemExit) as raised, - ): - self.invoke("--operator", "Ada") - self.assertEqual(raised.exception.code, 2) - self.assertIn("unrecognized arguments: --operator", stderr.getvalue()) - self.assertEqual(self.snapshot(), before) - self.extract.assert_not_called() - - def test_copy_stages_sidecars_references_annotations_and_options(self): - """Copy input trees and forward split policy/science overrides.""" - self.invoke( - "--stage-data", - "copy", - "--use-rois", - "--annotations-dir", - self.annotations, - "--max-workers", - 2, - "--max-trials", - 3, - "--verbose", - "--denoise-window-s", - 2, - "--vif", - 1.67, - ) - table, params = self.extract.call_args.args - self.assertEqual(table.datadr, self.results / "input") - for name in ("trial.dat", "trial.meta", "reference.tif"): - self.assertEqual( - (table.datadr / name).read_bytes(), - (self.data / name).read_bytes(), - ) - self.assertEqual( - (self.results / "annotations/annotations.h5").read_bytes(), - b"annotations", - ) - self.assertEqual(params, BandSiloParams(denoise_window_s=2, vif=1.67)) - self.assertEqual( - self.extract.call_args.kwargs, - { - "execution": ExecutionOptions(2, True, 3), - "annotations": AnnotationOptions(True, False), - }, - ) - - def test_symlink_staging_uses_correct_roots_without_link_privileges(self): - """Mock OS link creation while verifying real staging dispatch.""" - with patch.object(Path, "symlink_to", autospec=True) as link: - self.invoke("--stage-data", "symlink", "--stage-motion", "symlink") - self.assertEqual( - link.call_args_list, - [ - call( - self.results / "motion_correction", - self.motion, - target_is_directory=True, - ), - call( - self.results / "input", - self.data, - target_is_directory=True, - ), - ], - ) - table = self.extract.call_args.args[0] - self.assertEqual(table.datadr, self.results / "input") - self.assertEqual(table.savedr, self.results) - - def test_existing_directory_and_file_are_never_replaced(self): - """Existing contents survive even when output is empty or a file.""" - for destination in (self.data, self.table_path, self.scratch): - with self.subTest(destination=destination): - self.assert_rejected( - FileExistsError, "--results-dir", destination - ) - - def test_existing_and_dangling_results_links_are_never_replaced(self): - """Check live/dead link preflight without requiring link privilege.""" - exists, is_symlink = Path.exists, Path.is_symlink - for name, live in (("live", True), ("dead", False)): - link = self.root / name - with ( - self.subTest(name=name), - patch.object( - Path, - "exists", - autospec=True, - side_effect=lambda p: live if p == link else exists(p), - ), - patch.object( - Path, - "is_symlink", - autospec=True, - side_effect=lambda p: p == link or is_symlink(p), - ), - patch.object(Path, "unlink", autospec=True) as unlink, - ): - self.assert_rejected(FileExistsError, "--results-dir", link) - unlink.assert_not_called() - - def test_dangling_link_guard_without_os_link_privileges(self): - """The link predicate is checked even when exists() is false.""" - with ( - patch.object(Path, "exists", return_value=False), - patch.object(Path, "is_symlink", return_value=True), - self.assertRaises(FileExistsError), - ): - example.validate_destination(self.results, (self.data,)) - - def test_output_nested_in_any_input_is_rejected(self): - """Reject output within raw, motion, or annotation input roots.""" - for source in (self.data, self.motion, self.annotations): - with self.subTest(source=source): - self.assert_rejected( - ValueError, - "--results-dir", - source / "new/run", - "--annotations-dir", - self.annotations, - ) - - def test_missing_and_wrong_kind_paths_fail_before_staging(self): - """Check input paths and optional scratch roots before staging.""" - for flag, path, error in ( - ("--trial-table", self.root / "missing.h5", FileNotFoundError), - ("--trial-table", self.data, FileNotFoundError), - ("--data-dir", self.table_path, NotADirectoryError), - ("--data-dir", self.root / "absent", FileNotFoundError), - ("--motion-correction", self.scratch, FileNotFoundError), - ("--annotations-dir", self.scratch, FileNotFoundError), - ("--joblib-temp", self.table_path, NotADirectoryError), - ("--joblib-temp", self.root / "absent", FileNotFoundError), - ): - with self.subTest(flag=flag, path=path): - self.assert_rejected(error, flag, path) - self.load.assert_not_called() - - def test_missing_registration_metadata_fails_before_staging(self): - """A loaded table must identify SLAP2 data and alignment filenames.""" - for slap2, motion in ( - (None, {"fn_adata": [["alignment.h5"]]}), - (Slap2Info(), None), - (Slap2Info(), {}), - (Slap2Info(), {"alignHz": 80}), - ): - with self.subTest(slap2=slap2, motion=motion): - self.table.slap2_info = slap2 - self.table.motion_correction = motion - self.assert_rejected(ValueError) - - def test_unsafe_loaded_names_fail_before_staging(self): - """Both metadata fields are validated on the loaded table.""" - for field in ("filename", "fn_adata"): - with self.subTest(field=field): - self.table.filename = [["trial.dat"]] - self.table.motion_correction = {"fn_adata": [["alignment.h5"]]} - if field == "filename": - self.table.filename = [["../trial.dat"]] - else: - self.table.motion_correction[field] = [ - [r"..\alignment.h5"] - ] - self.assert_rejected(ValueError) - - def test_bad_science_and_execution_options_cannot_mutate_run(self): - """Validate finite positive settings before mkdir/copy/env changes.""" - for flag in ( - "--analyze-hz", - "--decay-tau-s", - "--baseline-window-s", - "--denoise-window-s", - "--vif", - "--d-xy", - "--peak-buffer", - "--psf-dilation", - "--max-workers", - "--max-trials", - ): - with self.subTest(flag=flag): - self.assert_rejected(ValueError, flag, 0) - for value in ("nan", "inf", "-1"): - with self.subTest(value=value): - self.assert_rejected(ValueError, "--analyze-hz", value) - - def test_invalid_roi_policy_fails_before_reading_table(self): - """Drawing requires ROI use; headless ROI use needs an input file.""" - for flags in (("--interactive",), ("--use-rois",)): - with self.subTest(flags=flags): - self.assert_rejected(SystemExit, *flags) - self.load.assert_not_called() - - def test_joblib_override_is_visible_only_during_execution(self): - """Restore both absent and pre-existing environment values.""" - for previous in (None, "", "original-scratch"): - for fails in (False, True): - with self.subTest(previous=previous, fails=fails): - destination = self.root / f"run-{previous}-{fails}" - with patch.dict(os.environ): - if previous is None: - os.environ.pop("JOBLIB_TEMP_FOLDER", None) - else: - os.environ["JOBLIB_TEMP_FOLDER"] = previous - environment = dict(os.environ) - - def extract(*args, **kwargs): - """Observe the worker setting in the backend.""" - self.assertEqual( - os.environ["JOBLIB_TEMP_FOLDER"], - str(self.scratch), - ) - if fails: - raise RuntimeError("backend failed") - return SimpleNamespace(paths=[]) - - self.extract.side_effect = extract - flags = ( - "--results-dir", - destination, - "--joblib-temp", - self.scratch, - ) - if fails: - with self.assertRaisesRegex( - RuntimeError, "backend failed" - ): - self.invoke(*flags) - else: - self.invoke(*flags) - self.assertEqual(dict(os.environ), environment) - self.assertTrue( - (destination / "input_trial_table.h5").is_file() - ) - self.assert_rejected( - FileExistsError, "--results-dir", destination - ) - - def test_no_joblib_flag_preserves_environment_during_extraction(self): - """Extraction observes the caller's environment without overrides.""" - environment = dict(os.environ) - - def extract(*args, **kwargs): - """Check the environment at the point workers would be launched.""" - self.assertEqual(dict(os.environ), environment) - return SimpleNamespace(paths=[]) - - self.extract.side_effect = extract - self.invoke() - self.assertEqual(dict(os.environ), environment) - - def test_destination_created_after_validation_is_not_overwritten(self): - """Exclusive mkdir refuses an output that appeared after preflight.""" - args = example.build_parser().parse_args(self.argv()) - run = example.prepare_run(args) - self.results.mkdir() - marker = self.results / "keep.txt" - marker.write_bytes(b"keep") - with ( - patch.object(example.shutil, "copy2") as copy, - self.assertRaises(FileExistsError), - ): - example.stage_and_extract(run, args) - copy.assert_not_called() - self.extract.assert_not_called() - self.assertEqual(marker.read_bytes(), b"keep") - - def test_stage_directory_guards_existing_destinations_and_invalid_mode( - self, - ): - """Staging never invokes copy/link on an existing or invalid target.""" - with ( - patch.object(example.shutil, "copytree") as copy, - patch.object(Path, "symlink_to") as link, - ): - for mode in ("copy", "symlink"): - with self.subTest(mode=mode): - with self.assertRaises(FileExistsError): - example.stage_directory(self.data, self.scratch, mode) - with ( - patch.object(Path, "exists", return_value=False), - patch.object(Path, "is_symlink", return_value=True), - self.assertRaises(FileExistsError), - ): - example.stage_directory(self.data, self.results, mode) - with self.assertRaisesRegex(ValueError, "Unknown staging mode"): - example.stage_directory(self.data, self.results, "invalid") - copy.assert_not_called() - link.assert_not_called() - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_workflows.py b/tests/test_workflows.py deleted file mode 100644 index 1f3a5b5..0000000 --- a/tests/test_workflows.py +++ /dev/null @@ -1,320 +0,0 @@ -"""Mocked service contracts: path identity, dispatch and API convergence.""" - -import sys -import unittest -from copy import deepcopy -from importlib import import_module -from pathlib import Path -from types import ModuleType -from unittest import mock - -import giant_python -from giant_python import cli -from giant_python.models import ( - AnnotationOptions, - BandSiloParams, - ExecutionOptions, - ExperimentSummary, - TrialTable, -) -from giant_python.pipeline import ( - Pipeline, - annotate_rois, - extract_band_sources, - extract_sources, -) - - -class TestPublicExports(unittest.TestCase): - """Lazy exports resolve to canonical objects and advertise real APIs.""" - - def test_lazy_exports_match_their_owner(self): - """Every advertised model and workflow resolves and is cached.""" - for name in giant_python.__all__: - with self.subTest(name=name): - owner = ( - "giant_python.models" - if name in giant_python._MODEL_EXPORTS - else "giant_python.pipeline" - ) - value = giant_python.__getattr__(name) - self.assertIs(value, getattr(import_module(owner), name)) - self.assertIs(giant_python.__dict__[name], value) - self.assertIn(name, dir(giant_python)) - - def test_unknown_export_has_standard_attribute_error(self): - """Unknown names raise AttributeError without importing a backend.""" - with self.assertRaisesRegex(AttributeError, "no attribute 'missing'"): - giant_python.__getattr__("missing") - - def test_legacy_apis_are_absent(self): - """Removed carriers and adapter classes have no reexport shims.""" - names = { - "giant_python.models.params": ( - "SiloParams", - "set_params", - "to_legacy_silo_params", - ), - "giant_python.models": ( - "SiloParams", - "set_params", - "to_legacy_silo_params", - ), - "giant_python.pipeline.extract": ( - "SourceExtractor", - "BandSourceExtractor", - "StandardSourceExtractor", - ), - "giant_python.pipeline": ( - "SourceExtractor", - "BandSourceExtractor", - "StandardSourceExtractor", - ), - "giant_python": ( - "SiloParams", - "SourceExtractor", - "BandSourceExtractor", - "StandardSourceExtractor", - ), - } - for module_name, removed in names.items(): - module = import_module(module_name) - for name in removed: - with self.subTest(module=module_name, name=name): - self.assertFalse(hasattr(module, name)) - - -class TestSharedServices(unittest.TestCase): - """All supported entry points converge without importing real backends.""" - - def setUp(self): - """Inject only the backend modules; keep public delegation real.""" - self.summary = ExperimentSummary() - self.extract = mock.Mock(return_value=self.summary) - self.annotate = mock.Mock(return_value="annotations.h5") - workflow = ModuleType("giant_python.extraction.band.workflow") - workflow.extract_band_sources = self.extract - annotation = ModuleType("giant_python.extraction.band.annotation") - annotation.annotate_band_rois = self.annotate - patch = mock.patch.dict( - sys.modules, - { - workflow.__name__: workflow, - annotation.__name__: annotation, - }, - ) - patch.start() - self.addCleanup(patch.stop) - - def test_direct_function_forwards_original_inputs(self): - """Scientific/run options are resolved once by the backend service.""" - table = TrialTable() - params = BandSiloParams() - execution = ExecutionOptions(max_workers=2) - annotations = AnnotationOptions(interactive=False) - result = extract_band_sources( - table, params, execution=execution, annotations=annotations - ) - self.assertIs(result, self.summary) - args, kwargs = self.extract.call_args - self.assertIs(args[0], table) - self.assertIs(args[1], params) - self.assertIs(kwargs["execution"], execution) - self.assertIs(kwargs["annotations"], annotations) - - def test_only_canonical_params_keyword(self): - """All public services accept params, not params_in.""" - for function in (extract_band_sources, extract_sources, annotate_rois): - with self.subTest(function=function.__name__): - function("custom.h5", params=BandSiloParams()) - with self.assertRaises(TypeError): - function("custom.h5", params_in={}) - - def test_generic_dispatch(self): - """No params, band params and explicit band all select band.""" - for params, mode in ( - (None, None), - (BandSiloParams(), None), - ({}, None), - (BandSiloParams(), "band"), - ): - with self.subTest(params=params, mode=mode): - self.assertIs( - extract_sources("custom.h5", params, scan_mode=mode), - self.summary, - ) - self.assertEqual(self.extract.call_count, 4) - - def test_explicit_unsupported_modes_are_never_ignored(self): - """Explicit routing is checked before backend invocation.""" - cases = ( - {"params": BandSiloParams(), "scan_mode": "standard"}, - {"scan_mode": "unknown"}, - {"microscope": "bergamo", "scan_mode": "band"}, - ) - for kwargs in cases: - with self.subTest(kwargs=kwargs): - with self.assertRaisesRegex(ValueError, "supported"): - extract_sources("custom.h5", **kwargs) - with self.assertRaisesRegex(ValueError, "supported"): - annotate_rois("custom.h5", **kwargs) - self.extract.assert_not_called() - self.annotate.assert_not_called() - - def test_missing_input_is_actionable(self): - """An omitted table fails before a heavy backend is invoked.""" - for function in (extract_band_sources, extract_sources, annotate_rois): - with self.subTest(function=function.__name__): - with self.assertRaisesRegex(ValueError, "trial-table"): - function(None) - self.extract.assert_not_called() - self.annotate.assert_not_called() - - def test_annotation_forwards_options_without_mutation(self): - """Annotation forwards split options directly to the backend.""" - table = TrialTable() - params = BandSiloParams() - options = AnnotationOptions(interactive=False) - before = deepcopy((params, options)) - result = annotate_rois(table, params, annotations=options) - self.assertEqual(result, "annotations.h5") - input, science = self.annotate.call_args.args - self.assertIs(input, table) - self.assertIs(science, params) - self.assertIs(self.annotate.call_args.kwargs["annotations"], options) - self.assertIsNone(self.annotate.call_args.kwargs["execution"]) - self.assertEqual((params, options), before) - - def test_pipeline_loads_nonstandard_filename_only_once(self): - """A table outside savedr is preserved through annotate/extract.""" - table = TrialTable(savedr=Path("/different/results")) - path = Path("/metadata/session_curated_v2.h5") - with mock.patch.object( - TrialTable, "from_h5", return_value=table - ) as load: - pipeline = Pipeline.from_trial_table(path) - result = pipeline.annotate().extract() - load.assert_called_once_with(path) - self.assertIs(result, pipeline) - self.assertIs(pipeline.summary, self.summary) - self.assertEqual(pipeline.trial_table_path, path) - self.assertEqual(table.source_path, path) - self.assertIs(self.annotate.call_args.args[0], table) - self.assertIs(self.extract.call_args.args[0], table) - self.assertTrue(self.extract.call_args.kwargs["annotations"].enabled) - - def test_in_memory_pipeline_never_guesses_source_path(self): - """savedr does not imply a trial_table.h5 exists in that directory.""" - for savedr in (None, Path("/results")): - with self.subTest(savedr=savedr): - table = TrialTable(savedr=savedr) - with mock.patch.object(TrialTable, "from_h5") as load: - pipeline = Pipeline.from_trial_table(table) - pipeline.extract() - load.assert_not_called() - self.assertIs(pipeline.tt, table) - self.assertIsNone(pipeline.trial_table_path) - self.assertIsNone(table.source_path) - self.assertIs(self.extract.call_args.args[0], table) - - def test_in_memory_known_source_path_is_retained(self): - """Metadata already loaded by IO retains its actual source path.""" - path = Path("/metadata/curated.h5") - table = TrialTable(source_path=path) - pipeline = Pipeline.from_trial_table(table) - self.assertIs(pipeline.tt, table) - self.assertEqual(pipeline.trial_table_path, path) - self.assertEqual(table.source_path, path) - - def test_annotation_chain_copies_policy(self): - """Explicit annotation enables later ROI use, not caller options.""" - options = AnnotationOptions(interactive=False) - pipeline = Pipeline.from_trial_table(TrialTable(), annotations=options) - pipeline.annotate().extract() - self.assertFalse(options.enabled) - self.assertTrue(pipeline.annotations.enabled) - self.assertIsNot(pipeline.annotations, options) - - def test_top_level_and_cli_share_backend_service(self): - """Direct, top-level, Pipeline and CLI paths reach the same driver.""" - self.assertIs(giant_python.extract_sources, extract_sources) - self.assertIs(giant_python.annotate_rois, annotate_rois) - result = giant_python.extract_sources("curated.h5") - self.assertIs(result, self.summary) - self.assertEqual( - cli.main( - [ - "extract", - "curated.h5", - "--headless", - "--max-workers", - "2", - ] - ), - 0, - ) - args, kwargs = self.extract.call_args - self.assertEqual(args[0], "curated.h5") - self.assertIsInstance(args[1], BandSiloParams) - self.assertEqual(kwargs["execution"].max_workers, 2) - self.assertFalse(kwargs["annotations"].interactive) - self.assertEqual(cli.main(["annotate", "curated.h5", "--headless"]), 0) - self.assertEqual(self.annotate.call_args.args[0], "curated.h5") - self.assertFalse( - self.annotate.call_args.kwargs["annotations"].interactive - ) - - -class TestPipelineErrors(unittest.TestCase): - """Unsupported or uninitialized pipeline use fails before side effects.""" - - def test_mixed_science_rejected_before_loading(self): - """Real services reject policy/routing fields rather than dispatch.""" - from giant_python.extraction.band import annotation, workflow - - with ( - mock.patch.object(workflow, "load_trial_table") as extract_load, - mock.patch.object(annotation, "load_trial_table") as roi_load, - ): - for field, value in ( - ("max_workers", 2), - ("operator", "Ada"), - ("draw_user_rois", True), - ("scan_mode", "band"), - ("microscope", "slap2"), - ): - params = {"peakth": 7.0, field: value} - for function in ( - extract_band_sources, - extract_sources, - annotate_rois, - ): - with self.subTest(field=field, api=function.__name__): - with self.assertRaisesRegex(TypeError, field): - function("curated.h5", params=params) - pipeline = Pipeline.from_trial_table( - TrialTable(), params=params - ) - for stage in (pipeline.extract, pipeline.annotate): - with self.subTest(field=field, stage=stage.__name__): - with self.assertRaisesRegex(TypeError, field): - stage() - extract_load.assert_not_called() - roi_load.assert_not_called() - - def test_missing_table(self): - """No table is a clear error, not an AttributeError on savedr.""" - pipeline = Pipeline() - for name in ("annotate", "extract"): - with self.subTest(stage=name): - with self.assertRaisesRegex(ValueError, "No trial table"): - getattr(pipeline, name)() - - def test_invalid_table_input(self): - """Invalid input types cannot be mistaken for paths.""" - with self.assertRaisesRegex(TypeError, "TrialTable"): - Pipeline.from_trial_table(None) - - -if __name__ == "__main__": - unittest.main() From 6b929a8c0f2de7141d00ea848ff1879f2caea2f7 Mon Sep 17 00:00:00 2001 From: m-xie Date: Thu, 22 Oct 2026 23:27:11 +0000 Subject: [PATCH 54/55] clean up remants of removed unittests --- .coverage-workflows | Bin 77824 -> 0 bytes .../workflows/{test_and_lint.yml => lint.yml} | 8 +++----- .github/workflows/tag_and_publish.yml | 16 +--------------- .gitignore | 13 +------------ CONTRIBUTING.md | 11 ++--------- docs/source/conf.py | 1 - pyproject.toml | 15 --------------- src/giant_python/cli.py | 2 +- src/giant_python/extraction/band/activity.py | 2 +- src/giant_python/extraction/band/annotation.py | 6 +++--- src/giant_python/extraction/band/background.py | 6 +++--- src/giant_python/extraction/band/traces.py | 4 ++-- src/giant_python/gui/parameters_gui.py | 2 +- src/giant_python/gui/roi_editor.py | 12 ++++++------ src/giant_python/io/hdf5.py | 3 +-- src/giant_python/numerics/peaks.py | 2 +- 16 files changed, 26 insertions(+), 77 deletions(-) delete mode 100644 .coverage-workflows rename .github/workflows/{test_and_lint.yml => lint.yml} (72%) diff --git a/.coverage-workflows b/.coverage-workflows deleted file mode 100644 index 4dd485a687de3008d124fab184031a0d260e589a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 77824 zcmeHQ33MCBdEQ+t?qhj~A}NX_MDY;CONVva5^0M%OiC0fK?em1T!Kpw5`cwqNTM8C zgyJ@F^IE4#-8QH4;+MuwWGl~(tvI$UIc<|ZCytY*wvzae@|@VQ9b0i@iL?%)|LiV+ z1wgq1cH7u_vya3q_W1UnZ~poJ`Db=8TYI~`5zao$hx}~BzL+Q=NRnu`+X;d&!+$aS zCpJCo(1|aQlCu5l?V5?!`+j0&wi6cJPJ-EMoyD|RJeK#&U$!haI?P^uv#}LU;0-f? z8Ndu+2L56gi1k|ZrRC-1(Xj|S=;I>c(Hf6yM}LiS;=kF$FNLtZx<@diBh$OxCXIvVDNkk5i+;I+_&t-}3i&*dG0yCR$i zAc7&UpAC)L_j9Ao!UfXPBHW>fcmO~+uP1=6u+NijZ?uQFVJ^f4++0|cVxD)X(bV|K zBPM-maWVNRPRJ7*a>JiFEjMr=m6eUd0OaOFLrD6;5F2ohaN%Y ze_MpN`@8`mn}M)5;^hN&?hxmWM!2DAGuVtsOKd=%JU4(ffHE{7-YGpx#!l+yY3`IR zmu?;g;R1Yu*3^yY(QvaMb0F#;1j_eC{XtYXaEJKd2!FjhuXk^Rmoj_9Wiw4#rt+Is zoC(TNiHdhc1KvBLT(a<+?WsGPg=_@)W;@CRXh5A=SGuB@BtR>azL&Aa#_CX&%n67Jr#)LVjAYcm;mkmb) zZd6smC=YJ_02g9CT&r|eGz))jMFQ1XXw;QjP^NWW6D%vS5Qn4`4?Z4zX}CJIF^Pg8rhf zf_fs~laf~iNPSOm4_7q`mqYd5!iU3NaI45|g;yqy+8ajLFtS472Kyk-`#3fr`UbdT zV;G+44ci47vc1xp6h?RyVR#NS2r?y{3o`TDpwcDk7wq``Y+y+8SI7;Es9^h1$go}9noE=>v z2qP%gK{jl6hd4MX`BFhJU~D8)DiLM_oJrVFF*l73nh(<*RK%`@=o3ZB2Go>`Rg?PC zrY15Lm1?kPoLpgO3HaGWUZt;8nr3C3Oet9pSZQb?$(0KcF5?h=6`z1CqFC&MB6W$Y zqz00y(F#(M%oPIu@P-+{3}6N@1DFBK0A>I)fEmCHUDp z1oJL@;0-f?8Ndu+1~3Dd0n7kq05gCYzzkppFawx@k0k>ZN>j-sUJF@JKx!(=rFQ_9 zwJu!Ry3hoyX z*`yFRu+!U06S0QdRyrMFm@?w@d)*@}=W9LW<(zAIcht`XB4OwHjU9n)En7w-BYeOa z4!NBkFB^!sg5m*B)ax7yu?Jlt-W!H_Bx_p$(wYlogbTw{(5K7K4{<((($x$o6?vk- zG^k;(CmMo@tb72#);9rIah_lY`OyAhAAb-@)juDAR^$SdMHfs!g~_={Fy|ml=y3{D zWD)j2BVc!`itXdwtj~KmLBq~@@Ps9*K0zw;y$yiep(-*P2=I}FmoPypI662N@HeVTOE?Pi%tE6sn8@uB_}Nnf z&sd}CGkBP8&PMor-o>G$(ewcl%3o&G`Az26h!qk$nPUS~Dn-=Zot9*&0@ZJ|+w zUZpxZvO5R75rp1d3FwuIWDkaTPlyYnQtY0s9E}aJJ|Fm=;An3Jpeb`oUf#(a0yTqq ziGc_}_RRvwMn%~LNU`z>a|;2ps~j+Ma#2~%0AYHEhY^EiiWvm3u#XKcLfI`ttpT)<^bQ`A}udwsks=nW>MU2g@zoEnx{Ccy3U zLbf|B0Hn`TG7-XI1_)&ZlE|5)^jruebC}eFP2eL~qk6D>$ioJ_hmrbhGXQK$E<9!x zNJ2<(AVNzW39DC6Sjng&VS9A|tIW?Og+)~eAh&A)a-pL9iy>Hsuj{21iz?v;!O#1V z)pKYRlPW=`dHQV>z$()K2^K;pPe5uvtRn%nFc)5BmLmv(c%chE3R)@nq!_eM6fq|l zm-eTc&r`FtMf9WQ8tN{(hdiUXpjmD-5`S+P(bpTEGR)FHryph9jK%hZjWzw)#G7Us ze_-_JerY{vvsqcI$?~YhsrwuA5lcBapZG2F1bGmF-|*H)#~R37@|9IzGUNZ&PC91I z2~tLtNIhsrOE(>>$QLCu{%_tu$BOd>OSCJH&Nl6)V=Hn3%AzaNJ0;`)`79mlR2MrV z{%_nx$Cjx31S$TX=b~dB>LRBapTK)VKOI}9E`B2ZuUEfFlktC@`n06Q|Fv#9wozSL zvf}@_d+FF3b)O-R|7+BzKNT77g`{9m<+j#Vm?T^9dW zs(_Y?|7R#s3iYGF^16JSPUGtK$&Vf|7Ovo=sJ^hkyU?hI=OP zg_r@%0A>I)fEmCHURzv^kaTXWj8?QYFv>DtmdIKQCo=)=X6Z+fVzr>;!J3$ISpPu{tp z7ETP-QW3~z?Ol5=w~Sz{!K_JXQvxNUVc z^-kSnPuzbmG4+d2-$|c;-FRl|OXqveAET?Q;Cy@4(ZfRC(}ah0)MB3dE2(+p zS~y!~nGIiO&89Ye5}qBbtAKCa71ZIbS=Zu6-hcbveky*!u#@fo$EkR{hZs0}A^fHD zHKcVG!kD$MuKg?IwKb-qa`;+bezePuG^GG7*KQ~)gJX-zsO=Onax}hPP?~X+J+iD6 zohg0mU;dVujK9+Ji8jO4YxTs$ojC~^?y33r>cqZ zMI~^0xP*G>jri#0!Xxj;*E7NRD^E_GES`!_?Rn$Kg#X;?Yb&OzpSf`DE_(HmmnPml zd9G*%+*3c}Xc2leln9h>`QBnw?l%|1ezZ2hbQ)83bhWUFN-vyjC&-j`j7#T*B%%4@ zF-l(q&nzgKPZ|qhXGYj+FNmX0nhIT9UwZD4z&LgKMHWM#BzUOlF*aHvV zv-hno;>@|?G7DU=$nxv&>^n8}>n|=L?t0O6TDNBG^4W$8Gn`p&rbdd1amZOMK`bJc z5QKL8_;JgeQWKn9U^=?=nD)#BPfVVE;jzb`zFhJrO{o!`Fj5b}Eq2Hflp9e#ZX()C z3~-{^uFVgMPEhyC@6zy9f;>?f1Yx@xuOLi@T9P&zyT7j|&h+np=&|8F`%gt*dFbLX$BENZ7caZOUY|XE zaX*E||J!U&5b)c7?=$Z(uQTVEUot;uo@M@(d5U?Ad6@Yo^EKuR%;%WTFn2R!%mIdH zMi`dqXSOokOeeFNS;j128kriVf|E|mwle^aH_QNL z05gCYzzkppFawwY%m8KpGk_Vu4CKcEioeOWHX&(k6_S<~A!%+FlBOmhnLl4h8XJXV z-aH{`Xb_V6dLgN+6O!6mA(=Z@NNQ??WX>ERvD<~Dx>`u8s)VGnQb=ac7LtkzA(=Hx zNXpBFq^wLxN=t=g=1d_eDG`zxGlZnLSV)SBgru-gND2ysgkgllW)l*tRY)urAu*eU z#AFf@qftlec*YQJ7M*UnM5_YcN z*b&&)vSl-X<|j* zTa~ZIEaW`kABIeKsIVv}%5>%uyzMzJXFGkahd4JMH~@mKQK`UX>-nx$zFzXjYT<2F zR|{2M9Kc;_byO%}b=E=8SQuP+5yNt;&Jgc)aYNn+9|Aes)WH-Mo8=;02rXcKi*h=V zoI@e@pbLaWkgXr}Wy~|z!j6mvIV5VB~$SAvVGSey#GA$FuQSw;=m` zHyzukuI%Bc-_M3dU0y%y5uP!37ad!o?lUBnnfHIX!1-P}wnkmC`5+f!p;98PsZqaF z{d~mB2LO8xOUF9Z72U_XS)cbXD?Oo6ed)b{U^If5sZ-w&csSnAMM9&1Uafv@9c116 zJt00C7y^8|`uHgq0r(B-v%|V0-UHqUVy9{Y9V=F*Ic^_X?%TSPjxAOOG{^>q!d@Tm z6qiqztl`bu=vb3#z>*nUrQDDtrOmQ`E0wp?li3F{4VK5qa;70Z>k z7E;o3?0zKbvR*n?r&5`TVZc)5?T@6CBSXVWK*%L33Mnpmf0K$OB(q?PRfLsA7mHM^ zv1t}@p^EZKrVuIW0u$*Y=8`pp=8u~DH9gc->f6*-N=rUwylDKou^WEnkFY&p>(#Yb zFIvB>eaX7P@{aay%Q4Fa^F{M9^9Iv!VLs00RGz*X?3M?h$WzKw zUu2lvA=V%CA(-tO=~zj=FsORC5=~r(?kJI4Gf^f`qN2Hw9mkyuVW;2g9$`6O>me`a zT+6#}yz>WlF65-mSv@om!H0cpa1qpvvYh*SrqgJdRqj#+P-c&B&)LOICt;c;-l&R< zG8?=>xoMY7U|8O>xn8*|PQr@qg>}kXjY-60E$n2zMLnCYk?ZMLVfvB(rJ~Eq40W*h zd|(%XqkD5UlW92a{SbP)AmU-AM)Mve1uuq`F4V6Hz0_iax%)OcX3RIEs5=)j*}s{N zRp*Q02T_x`37Pq)x^B%Uq*g1(ixUgxURcY?-nOn)F{8<9#!74q&osX%ady=2xsdmP}&$ z`A~3#_wgRoKUJi>-!TcCIRsd!ToIDkvUXg73L}=HZ58{$BJ7|L-D#Cq1;2`1%msk|Q zR97>gROE>Q&Awr;Cko>hV#{xR6Mz-x2_|ph^v?&N6}bRq(RJNnt${|s?o<^!V*q65 zJb1zqRi7XYYVBj_UL$hZOhPRi{6#Yr&;s0oYZFU^81-F!mhv32$li*Z_QP zF5t4JX>v#?5sZVPt+xVTP7TW}6Cl2N&|v`}eV&qu5DqgyC@YXe&g4Bf4wHJY34CDA zVN?%R)`PPR)-Q~;Ip0BZt2wltTI2B6jtgO+O7r2g^KboMZ@WV zdoQh6)Xc!$p;1igbb-H+i zeX3!%c@N;s$y=tZA)-Df;N{e+la-3(*)oS@dUgY5zCL2QVW(UB0WjZD5oy?I(=NcM z&Xc(8VW;+;0H@6x4)hqNH01BE#9^n-?Et~#DMadpkqr)jDajKi?S+xvZGe=MIhJWt zvUQLapl{j=kolII3{raK&(Q~PIl1`hB+Lr&dwKyjUzJZvDGLX;ZUMk11!aO>p>zdD zV6ekA?DfI>`=D~&Jph_h#5!H(6K(F>Hv?L}<}sOPaiG3y6JX^O2u>%N6dB##t%hJ? z73=8&EX4&Y>yzlYT@654bH8^Z04lC?X}vN%8`O|d)=YD(2f&v@99X zC{x;Ot{pc6rm{|xgc(nNS%2#)09ICuWWbW=-n|k)m33<*L9_g~V+DXKGf+uznHP5~ zR}nV#c2D0jfK{dpDQTsb^|veqOl7t-i78gs-X(yjtQI5@CC|R)CiR3A{kvl^AgWaG zGB58~q#mry3hY=2uqxHqG>`9Cpq{)kzu(seu*wt_S*^5rt!Vy#x#6D)=26DOl-vHz z7Pi$`e{B7PwbAmrdLQ}#dW>=7N#lNFt>HIt3*ImTm;uZHW&ksQ z8Ndvv%0SLjsAWA^iGg=v3V@R72%#wYA%FpuW)Z0Q+U7@ficHS0LCcEmI^!Qs)u-Lu z;B=1EJPnzU;!qrDMGKz1gMMI3MvWK(?oeskZO*@2B$Fibb=zhoMnHU-!IJZf;xc&> zfwwSuH~Et^+muY*l0iy6I$Lvo?|eE5v-)!SlvKZ@l(Jr&UUdLd{WrFprzK98@r=Qp zEjiCTnHE&)4uWxy{9oBmk|=frZB|kz>% diff --git a/.github/workflows/test_and_lint.yml b/.github/workflows/lint.yml similarity index 72% rename from .github/workflows/test_and_lint.yml rename to .github/workflows/lint.yml index ef1c276..d1033e0 100644 --- a/.github/workflows/test_and_lint.yml +++ b/.github/workflows/lint.yml @@ -1,4 +1,4 @@ -name: Lint and run tests +name: Lint on: pull_request: @@ -18,9 +18,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install dependencies - run: | + run: | python -m pip install -e . --group dev --no-cache-dir - name: Run linter checks - run: flake8 . && interrogate --verbose . - - name: Run tests and coverage - run: coverage run -m unittest discover && coverage report + run: flake8 . && interrogate --verbose . \ No newline at end of file diff --git a/.github/workflows/tag_and_publish.yml b/.github/workflows/tag_and_publish.yml index 3f8c804..b729b2e 100644 --- a/.github/workflows/tag_and_publish.yml +++ b/.github/workflows/tag_and_publish.yml @@ -3,8 +3,7 @@ on: push: branches: - main -# Remove line 65 to enable automated semantic version bumps. -# Change line 71 from "if: false" to "if: true" to enable PyPI publishing. +# Set the publish job's "if" condition to true to enable PyPI publishing. # Requires that svc-aindscicomp be added as an admin to repo. jobs: update_badges: @@ -41,19 +40,6 @@ jobs: new_interrogate_badge="interrogate-$interrogate_val%25-yellow" fi sed -i "s/$interrogate_badge/$new_interrogate_badge/g" README.md - - name: Get Coverage values and Update README.md - run: | - coverage run -m unittest discover - coverage_val=$(coverage report | grep "^TOTAL" | grep -o '[0-9]\+%' | grep -o '[0-9]\+') - coverage_badge=$(grep '^\!\[Coverage\]' README.md | sed -n 's/.*\(coverage-[^)]*\).*/\1/p') - if (( $(echo "$coverage_val >= 90.00" | bc -l) )); then - new_coverage_badge="coverage-$coverage_val%25-brightgreen" - elif (( $(echo "$coverage_val < 80.00" | bc -l) )); then - new_coverage_badge="coverage-$coverage_val%25-red" - else - new_coverage_badge="coverage-$coverage_val%25-yellow" - fi - sed -i "s/$coverage_badge/$new_coverage_badge/g" README.md - name: Commit changes uses: EndBug/add-and-commit@v10 with: diff --git a/.gitignore b/.gitignore index 649f1f1..405b401 100644 --- a/.gitignore +++ b/.gitignore @@ -37,19 +37,8 @@ MANIFEST pip-log.txt pip-delete-this-directory.txt -# Unit test / coverage reports -htmlcov/ -.tox/ -.nox/ -.coverage -.coverage.* +# Tool caches .cache -nosetests.xml -coverage.xml -*.cover -*.py,cover -.hypothesis/ -.pytest_cache/ # Translations *.mo diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a53808c..1f16d22 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -10,15 +10,9 @@ pip install -e . --group dev Alternatively, use `uv sync`. Interactive ROI work also requires tkinter in the Python installation. -### Linters and testing +### Code quality -There are several libraries used to run linters, check documentation, and run tests. - -- Please test your changes using the **coverage** library, which will run the tests and log a coverage report: - -```bash -coverage run -m unittest discover && coverage report -``` +Use the following tools to check documentation and maintain code style. - Use **interrogate** to check that modules, methods, etc. have been documented thoroughly: @@ -57,7 +51,6 @@ where scope (optional) describes the packages affected by the code changes and t - **fix**: A bugfix - **perf**: A code change that improves performance - **refactor**: A code change that neither fixes a bug nor adds a feature -- **test**: Adding missing tests or correcting existing tests ### Semantic Release diff --git a/docs/source/conf.py b/docs/source/conf.py index ff5ee46..ca4787c 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -29,7 +29,6 @@ extensions = [ "sphinx.ext.duration", - "sphinx.ext.doctest", "sphinx.ext.autodoc", "sphinx.ext.napoleon", "sphinx.ext.intersphinx", diff --git a/pyproject.toml b/pyproject.toml index c8cb56f..f23911b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,6 @@ giant = "giant_python.cli:main" [dependency-groups] dev = [ 'black', - 'coverage', 'flake8', 'interrogate', 'isort', @@ -70,7 +69,6 @@ exclude = ''' | \.git # root of the project | \.hg | \.mypy_cache - | \.tox | \.venv | _build | build @@ -81,19 +79,6 @@ exclude = ''' ) ''' -[tool.coverage.run] -omit = ["*__init__*"] -source = ["giant_python", "tests"] - -[tool.coverage.report] -exclude_lines = [ - "if __name__ == .__main__.:", - "^from .* import .*", - "^import .*", - "pragma: no cover" -] -fail_under = 100 - [tool.isort] line_length = 79 profile = "black" diff --git a/src/giant_python/cli.py b/src/giant_python/cli.py index c22007c..9dcd815 100644 --- a/src/giant_python/cli.py +++ b/src/giant_python/cli.py @@ -215,5 +215,5 @@ def main(argv: Optional[Sequence[str]] = None) -> int: parser.error(str(exc)) -if __name__ == "__main__": # pragma: no cover +if __name__ == "__main__": raise SystemExit(main()) diff --git a/src/giant_python/extraction/band/activity.py b/src/giant_python/extraction/band/activity.py index b302c0f..6427e38 100644 --- a/src/giant_python/extraction/band/activity.py +++ b/src/giant_python/extraction/band/activity.py @@ -180,7 +180,7 @@ def compute_rho( # Unreachable defensive guard (mirrors the reference): a plane # with selected pixels (new_ncols > 0) always has H columns in # it, since both derive from the same in-plane PSF footprint. - continue # pragma: no cover + continue remap = z_remaps[z] new_rows = h_idxs[0, keep_mask] diff --git a/src/giant_python/extraction/band/annotation.py b/src/giant_python/extraction/band/annotation.py index 442115c..39a7218 100644 --- a/src/giant_python/extraction/band/annotation.py +++ b/src/giant_python/extraction/band/annotation.py @@ -49,7 +49,7 @@ def motion_median_from_adata(a_data: dict) -> tuple: ) -def _stdin_is_tty() -> bool: # pragma: no cover - runtime stdin +def _stdin_is_tty() -> bool: """Check for interactive stdin without importing a GUI toolkit.""" return bool(getattr(sys.stdin, "isatty", lambda: False)()) @@ -131,7 +131,7 @@ def _draw_and_save_user_rois( n_dmds: int, user_roi_geo: dict, ref_files: Optional[dict], -) -> dict: # pragma: no cover - GUI +) -> dict: """Launch the lazy ROI editor and save its records and masks.""" from ...gui.roi_editor import run_user_roi_selection @@ -179,7 +179,7 @@ def annotate_band_rois( *, execution: ExecutionInput = None, annotations: AnnotationInput = None, -) -> str: # pragma: no cover - full reference IO and optional GUI +) -> str: """Annotate a path or in-memory TrialTable; return annotations.h5's path. Existing usable annotations are loaded headlessly. No extraction or diff --git a/src/giant_python/extraction/band/background.py b/src/giant_python/extraction/band/background.py index 207009a..cb1cdaf 100644 --- a/src/giant_python/extraction/band/background.py +++ b/src/giant_python/extraction/band/background.py @@ -8,9 +8,9 @@ import pandas as pd from scipy.interpolate import CubicSpline -try: # pragma: no cover - exercised via the installed extra +try: import bottleneck as bn -except ImportError: # pragma: no cover +except ImportError: bn = None @@ -107,7 +107,7 @@ def _interp_columns( # noqa: C901 # Unreachable defensive guard (mirrors the reference): sorted # distinct rows make searchsorted never straddle equal rows, so # denom is always non-zero here. - continue # pragma: no cover + continue interp_rows = interp_rows[nonzero] lo_v = lo_v[nonzero] diff --git a/src/giant_python/extraction/band/traces.py b/src/giant_python/extraction/band/traces.py index e405335..bf6b3cd 100644 --- a/src/giant_python/extraction/band/traces.py +++ b/src/giant_python/extraction/band/traces.py @@ -18,7 +18,7 @@ The pure compute (:func:`compute_high_res_traces`) is factored out of the IO wrapper (:func:`get_high_res_traces`, which reads the SLAP2 file lazily via -:mod:`giant_python.extraction.band.trial_data`) so the numerics are testable +:mod:`giant_python.extraction.band.trial_data`) so the numerics can be used without ``slap2_utils``. The workflow owns per-trial scheduling. Results use ``TrialTraceResult``, with denoised traces, events and the residual backprojection after the original fields. @@ -634,7 +634,7 @@ def _load_high_res_trial_data( datadr: str, trial_table: dict, num_channels: int, -): # pragma: no cover - requires slap2_utils +): """Read one trial's high-res activity via the SLAP2 reader. Returns diff --git a/src/giant_python/gui/parameters_gui.py b/src/giant_python/gui/parameters_gui.py index 47d40eb..fe1b3c9 100644 --- a/src/giant_python/gui/parameters_gui.py +++ b/src/giant_python/gui/parameters_gui.py @@ -34,7 +34,7 @@ def run_parameter_gui( *, execution: ExecutionInput = None, annotations: AnnotationInput = None, -) -> Optional[ # pragma: no cover - interactive tkinter dialog +) -> Optional[ tuple[BandSiloParams, ExecutionOptions, AnnotationOptions] ]: """Return split option objects, or None on cancellation, without mutation. diff --git a/src/giant_python/gui/roi_editor.py b/src/giant_python/gui/roi_editor.py index 8af47b2..c56d442 100644 --- a/src/giant_python/gui/roi_editor.py +++ b/src/giant_python/gui/roi_editor.py @@ -13,7 +13,7 @@ def _ask_roi_label( roi_num: int, path_num: int, plane_num: int, -) -> str: # pragma: no cover - interactive tkinter dialog +) -> str: """Prompt for one ROI label, defaulting to ROI{n} on cancel or failure.""" import tkinter as tk from tkinter import simpledialog @@ -39,7 +39,7 @@ def _ask_roi_label( def run_user_roi_selection( user_roi_geo: dict, n_dmds: int -) -> dict: # pragma: no cover - GUI +) -> dict: """Select ROIs on reference images and return per-DMD selections.""" import cv2 @@ -68,7 +68,7 @@ def run_user_roi_selection( def _normalize_plane( plane: np.ndarray, -) -> np.ndarray: # pragma: no cover - display +) -> np.ndarray: """Percentile-normalize a reference plane to an 8-bit image.""" im = np.nan_to_num(plane, nan=0.0) vmin = np.percentile(im, 1) @@ -83,7 +83,7 @@ def _normalize_plane( return (im8 * 255).astype(np.uint8) -def _select_rois_for_dmd(cv2, dmd_ix: int, g: dict): # pragma: no cover - GUI +def _select_rois_for_dmd(cv2, dmd_ix: int, g: dict): """Run the per-DMD interactive loop, returning masks and records.""" ref = g["ref"] num_ref_z = g["num_ref_z"] @@ -143,7 +143,7 @@ def _select_rois_for_dmd(cv2, dmd_ix: int, g: dict): # pragma: no cover - GUI def _compose_display( cv2, im8, sp_plane, roi_plane -): # pragma: no cover - display +): """Overlay the superpixel mask in green and ROI union in red.""" disp = cv2.cvtColor(im8, cv2.COLOR_GRAY2BGR) if np.any(sp_plane): @@ -168,7 +168,7 @@ def _edit_rois( roi_masks, roi_union, roi_records, -): # pragma: no cover - GUI +): """Append selected rectangles as polygon records and overlapping masks.""" edit_name = f"Edit ROIs z={curr_fz + 1}" edit_disp = _compose_display( diff --git a/src/giant_python/io/hdf5.py b/src/giant_python/io/hdf5.py index bdaabc3..9e3fedd 100644 --- a/src/giant_python/io/hdf5.py +++ b/src/giant_python/io/hdf5.py @@ -1,8 +1,7 @@ """MATLAB-style HDF5 decoding and experiment-parameter group writing. This module is the foundation of MATLAB interoperability: the model -``from_h5`` / ``to_h5`` methods build on these helpers. Round-trip tests -against real MATLAB-produced ``.h5`` files are the acceptance criterion. +``from_h5`` / ``to_h5`` methods build on these helpers. Serialization conventions ------------------------- diff --git a/src/giant_python/numerics/peaks.py b/src/giant_python/numerics/peaks.py index 65aa78a..4c63715 100644 --- a/src/giant_python/numerics/peaks.py +++ b/src/giant_python/numerics/peaks.py @@ -18,7 +18,7 @@ The reference's nested helpers (``_make_bounds``/``_peak_mask``/ ``_buffer_mask`` and the per-component new-peak step) are lifted to -module-level functions here so each is independently testable. +module-level functions for independent reuse. """ from __future__ import annotations From 5dc4d10cd4840c68b826b6afed3e188f051a2f1a Mon Sep 17 00:00:00 2001 From: m-xie Date: Thu, 22 Oct 2026 23:45:17 +0000 Subject: [PATCH 55/55] fix formatting --- docs/source/conf.py | 4 +- examples/extract_band.py | 8 +- src/giant_python/cli.py | 33 ++- src/giant_python/extraction/band/activity.py | 3 +- .../extraction/band/result_assembly.py | 43 ++-- src/giant_python/extraction/band/traces.py | 47 +++-- src/giant_python/extraction/band/transient.py | 193 ++++++++++++------ src/giant_python/extraction/band/workflow.py | 7 +- src/giant_python/gui/parameters_gui.py | 12 +- src/giant_python/gui/roi_editor.py | 8 +- src/giant_python/io/experiment_summary.py | 57 +++--- src/giant_python/models/experiment_summary.py | 9 +- src/giant_python/models/params.py | 6 +- 13 files changed, 288 insertions(+), 142 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index ca4787c..9fd81d3 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -53,7 +53,9 @@ "BandSiloParams": "giant_python.models.params.BandSiloParams", "ExecutionOptions": "giant_python.models.params.ExecutionOptions", "AnnotationOptions": "giant_python.models.params.AnnotationOptions", - "ExperimentSummary": "giant_python.models.experiment_summary.ExperimentSummary", + "ExperimentSummary": ( + "giant_python.models.experiment_summary.ExperimentSummary" + ), } diff --git a/examples/extract_band.py b/examples/extract_band.py index a18eb75..ce7c805 100644 --- a/examples/extract_band.py +++ b/examples/extract_band.py @@ -109,8 +109,12 @@ def build_parser() -> argparse.ArgumentParser: parser.add_argument("--d-xy", type=int, default=5) parser.add_argument("--peak-buffer", type=int, default=3) parser.add_argument("--psf-dilation", type=int, default=17) - parser.add_argument("--z-tol", type=float, default=1.5, - help="Z tolerance for motion-bin selection (microns)") + parser.add_argument( + "--z-tol", + type=float, + default=1.5, + help="Z tolerance for motion-bin selection (microns)", + ) return parser diff --git a/src/giant_python/cli.py b/src/giant_python/cli.py index 9dcd815..502e5f0 100644 --- a/src/giant_python/cli.py +++ b/src/giant_python/cli.py @@ -20,19 +20,38 @@ def _add_silo_options(parser: argparse.ArgumentParser) -> None: """Attach the shared source-extraction options to a subparser.""" for name, kind, help_text in ( - ("decay_tau_s", float, "Shared localization and trace decay time in seconds."), - ("trace_beta_fraction", float, "Trial-relative nonnegative event penalty."), + ( + "decay_tau_s", + float, + "Shared localization and trace decay time in seconds.", + ), + ( + "trace_beta_fraction", + float, + "Trial-relative nonnegative event penalty.", + ), ("trace_max_iter", int, "Maximum iterations per transient fit stage."), ("trace_tolerance", float, "Transient optimizer tolerance."), - ("trace_event_threshold", float, "Absolute event threshold for debias support."), - ("random_seed", int, "Nonnegative seed for the localization motion order."), + ( + "trace_event_threshold", + float, + "Absolute event threshold for debias support.", + ), + ( + "random_seed", + int, + "Nonnegative seed for the localization motion order.", + ), ): parser.add_argument( - "--" + name.replace("_", "-"), type=kind, - default=getattr(BandSiloParams, name), help=help_text, + "--" + name.replace("_", "-"), + type=kind, + default=getattr(BandSiloParams, name), + help=help_text, ) parser.add_argument( - "--trace-debias", action="store_true", + "--trace-debias", + action="store_true", help="Refit selected transient events without the L1 penalty.", ) parser.add_argument( diff --git a/src/giant_python/extraction/band/activity.py b/src/giant_python/extraction/band/activity.py index 6427e38..3422843 100644 --- a/src/giant_python/extraction/band/activity.py +++ b/src/giant_python/extraction/band/activity.py @@ -68,7 +68,8 @@ def _valid_sel_cols_for_motion( ) valid_pix_mask = ndimage.binary_erosion( valid_pix_mask, - # Erode each plane with PSF-height rows and (2 * PSF-width - 1) columns. + # Erode each plane with PSF-height rows and + # (2 * PSF-width - 1) columns. structure=np.ones( (1, psf2d.shape[0], psf2d.shape[1] * 2 - 1), dtype=bool ), diff --git a/src/giant_python/extraction/band/result_assembly.py b/src/giant_python/extraction/band/result_assembly.py index d76ed6d..41fb7ac 100644 --- a/src/giant_python/extraction/band/result_assembly.py +++ b/src/giant_python/extraction/band/result_assembly.py @@ -83,10 +83,13 @@ def assemble_path_summary(pr: PathResult, path_index: int = 0) -> PathSummary: results = pr.trace_results d_f = np.concatenate([r.d_f for r in results], axis=0) # F = dF_ls + intermediate F0 + backprojected fit residual. - f0_ls = np.concatenate([ - r.f0_ls if r.f_residual is None else r.f0_ls + r.f_residual - for r in results - ], axis=0) + f0_ls = np.concatenate( + [ + r.f0_ls if r.f_residual is None else r.f0_ls + r.f_residual + for r in results + ], + axis=0, + ) f_full, f0, d_f, _ = assemble_dff( d_f, f0_ls, pr.denoise_window, pr.baseline_window ) @@ -95,18 +98,28 @@ def assemble_path_summary(pr: PathResult, path_index: int = 0) -> PathSummary: ) coords = pr.source_params[:, :3].numpy() df_channels = _with_nan_channel(d_f, pr.num_channels) - # Keep the fitted convolution trace itself, not a second baseline-subtracted - # version: otherwise it would no longer correspond to the fitted events. + # Keep the fitted convolution trace itself, not a second + # baseline-subtracted version: otherwise it would no longer correspond + # to the fitted events. # Legacy manually constructed records have no transient outputs. - denoised = np.concatenate([ - r.d_f_denoised if r.d_f_denoised is not None - else np.full_like(r.d_f, np.nan) - for r in results - ], axis=0) - events = np.concatenate([ - r.events if r.events is not None else np.full_like(r.d_f, np.nan) - for r in results - ], axis=0) + denoised = np.concatenate( + [ + ( + r.d_f_denoised + if r.d_f_denoised is not None + else np.full_like(r.d_f, np.nan) + ) + for r in results + ], + axis=0, + ) + events = np.concatenate( + [ + r.events if r.events is not None else np.full_like(r.d_f, np.nan) + for r in results + ], + axis=0, + ) denoised_channels = _with_nan_channel(denoised, pr.num_channels) event_channels = _with_nan_channel(events, pr.num_channels) f0_channels = _with_nan_channel(f0, pr.num_channels) diff --git a/src/giant_python/extraction/band/traces.py b/src/giant_python/extraction/band/traces.py index bf6b3cd..a5830a2 100644 --- a/src/giant_python/extraction/band/traces.py +++ b/src/giant_python/extraction/band/traces.py @@ -229,6 +229,25 @@ def _backproject(x: torch.Tensor, data: torch.Tensor) -> torch.Tensor: return (data.T @ x) / norm.masked_fill(norm <= 0, 1.0) +def _validate_transient_settings( + transient_options, simple_trace_extraction, analyze_hz +): + """Validate mutually exclusive trace modes and the transient grid rate.""" + if transient_options is not None: + if simple_trace_extraction: + raise ValueError( + "Simple and convolution trace extraction are mutually " + "exclusive" + ) + if not isinstance(transient_options, TransientOptions): + raise TypeError( + "transient_options must be TransientOptions or None" + ) + from ...models.params import _positive + + _positive("analyze_hz", analyze_hz) + + def _solve_trial_phi_f0( data: np.ndarray, background: np.ndarray, @@ -303,14 +322,9 @@ def _solve_trial_phi_f0( least-squares fit. The optional residual term is the backprojection of ``data - background - x @ phi`` (zero for simple extraction). """ - if transient_options is not None: - if simple_trace_extraction: - raise ValueError("Simple and convolution trace extraction are mutually exclusive") - if not isinstance(transient_options, TransientOptions): - raise TypeError("transient_options must be TransientOptions or None") - from ...models.params import _positive - - _positive("analyze_hz", analyze_hz) + _validate_transient_settings( + transient_options, simple_trace_extraction, analyze_hz + ) n_sources = a_final.shape[1] n_frames = data.shape[1] phi = torch.full((n_frames, n_sources), float("nan"), dtype=torch.float32) @@ -334,7 +348,9 @@ def _solve_trial_phi_f0( ) extract = ( - _weighted_trace_average if simple_trace_extraction else solve_phi_motion + _weighted_trace_average + if simple_trace_extraction + else solve_phi_motion ) designs = {} for motion_idx in mot_inds_to_keep: @@ -365,7 +381,11 @@ def _solve_trial_phi_f0( # Preserve the empty/all-invalid behavior of the existing extractor. if np.any(valid): result = fit_transient_traces( - residual32, designs, mot_inds, valid, analyze_hz, + residual32, + designs, + mot_inds, + valid, + analyze_hz, options=transient_options, ) if not result.metadata["converged"]: @@ -528,7 +548,8 @@ def compute_high_res_traces( Add constrained dF and events, retaining QR dF, backprojected F0 and auxiliary data. Fits nonnegative sparse events plus an unpenalized initial state. - Public dF retains missing-frame NaNs; nonconvergence raises RuntimeError. + Public dF retains missing-frame NaNs; nonconvergence raises + RuntimeError. analyze_hz : float or None Required nominal rate for transient fitting on the uniform full grid. Acquisition averaging is approximated, not explicitly modeled. @@ -548,7 +569,9 @@ def compute_high_res_traces( # can have alternating line spacings. Use the explicit nominal rate. steps = np.diff(frame_grid) if np.any(steps <= 0): - raise ValueError("Convolution traces require chronological frames; retain gaps") + raise ValueError( + "Convolution traces require chronological frames; retain gaps" + ) ( motion_r, motion_c, diff --git a/src/giant_python/extraction/band/transient.py b/src/giant_python/extraction/band/transient.py index 4d41924..4b9c921 100644 --- a/src/giant_python/extraction/band/transient.py +++ b/src/giant_python/extraction/band/transient.py @@ -7,8 +7,8 @@ Reader averaging is not modeled exactly; events are not spike times. """ -from dataclasses import dataclass import warnings +from dataclasses import dataclass import numpy as np from scipy.optimize import minimize @@ -50,7 +50,7 @@ def __post_init__(self): @dataclass class TransientResult: - """Trace, latent predictions/events, and explicit convergence diagnostics.""" + """Trace, latent predictions/events, and convergence diagnostics.""" d_f: np.ndarray metadata: dict @@ -63,25 +63,31 @@ class TransientOperator: """Causal convolution and exact adjoint without a dense temporal matrix.""" def __init__(self, analyze_hz, tau_s, kernel=None): - """Use a unit-peak exponential or an explicit nonnegative causal kernel.""" + """Use a unit-peak exponential or a nonnegative causal kernel.""" _positive("analyze_hz", analyze_hz) _positive("tau_s", tau_s) self.gamma = float(np.exp(-1 / (analyze_hz * tau_s))) - self.kernel = None if kernel is None else np.asarray(kernel, dtype=float) + self.kernel = ( + None if kernel is None else np.asarray(kernel, dtype=float) + ) if self.kernel is not None and ( - self.kernel.ndim != 1 or not self.kernel.size + self.kernel.ndim != 1 + or not self.kernel.size or not np.all(np.isfinite(self.kernel)) - or np.any(self.kernel < 0) or self.kernel[0] <= 0 + or np.any(self.kernel < 0) + or self.kernel[0] <= 0 ): - raise ValueError("kernel must be finite, nonnegative, with kernel[0]>0") + raise ValueError( + "kernel must be finite, nonnegative, with kernel[0]>0" + ) def forward(self, events): """Convolve all sources in chronological order, including gaps.""" if self.kernel is None: return lfilter([1.0], [1.0, -self.gamma], events, axis=0) - return fftconvolve( - events, self.kernel[:, None], mode="full", axes=0 - )[:len(events)] + return fftconvolve(events, self.kernel[:, None], mode="full", axes=0)[ + : len(events) + ] def adjoint(self, traces): """Apply the exact transpose, including finite-trial boundaries.""" @@ -101,25 +107,8 @@ def _temporal_quadratic(normal, groups, trace): return float(value), gradient -def fit_transient_traces( - residual, designs, motion_indices, valid, analyze_hz, - *, options=None, kernel=None, -): - """Fit one trial using fixed motion-specific (superpixel, source) designs. - - Residual is (superpixels, frames). The complete nominal output grid must - be retained, in chronological order: never compress out gaps or group - disjoint motion frames before convolution. Nonfinite observations reject - the entire frame, as in QR. ``valid`` also excludes unmatched motion/z. - Public dF is NaN there, while model_trace contains latent gap predictions. - - Low-level nonconvergence returns diagnostics and warns. Production callers - must check ``metadata['converged']`` before saving a result. - """ - options = TransientOptions() if options is None else options - if not isinstance(options, TransientOptions): - raise TypeError("options must be TransientOptions or None") - op = TransientOperator(analyze_hz, options.tau_s, kernel) +def _prepare_transient_system(residual, designs, motion_indices, valid): + """Validate arrays and build motion-specific sufficient statistics.""" residual = np.asarray(residual) motion_indices = np.asarray(motion_indices) valid = np.asarray(valid, dtype=bool).copy() @@ -127,7 +116,9 @@ def fit_transient_traces( raise ValueError("residual must have shape (superpixels, frames)") n_sp, n_frames = residual.shape if valid.shape != (n_frames,) or motion_indices.shape != (n_frames,): - raise ValueError("motion_indices and valid must have one entry per frame") + raise ValueError( + "motion_indices and valid must have one entry per frame" + ) if not designs: raise ValueError("Transient fitting requires projected profiles") arrays = {m: np.asarray(x) for m, x in designs.items()} @@ -137,11 +128,15 @@ def fit_transient_traces( n_sources = first.shape[1] for x in arrays.values(): if x.shape != (n_sp, n_sources) or not np.all(np.isfinite(x)): - raise ValueError("Projected profiles must be finite with matching shapes") + raise ValueError( + "Projected profiles must be finite with matching shapes" + ) valid &= np.all(np.isfinite(residual), axis=0) valid &= np.isin(motion_indices, list(arrays)) if not np.any(valid) or n_sources == 0: - raise ValueError("Transient fitting requires observed frames and sources") + raise ValueError( + "Transient fitting requires observed frames and sources" + ) groups, normal = {}, {} for motion, x in arrays.items(): ix = np.flatnonzero(valid & (motion_indices == motion)) @@ -149,24 +144,57 @@ def fit_transient_traces( xd = x.astype(np.float64) groups[motion] = ix normal[motion] = ( - xd.T @ xd, xd.T @ residual[:, ix].astype(np.float64) + xd.T @ xd, + xd.T @ residual[:, ix].astype(np.float64), ) + return (n_frames, n_sources), valid, groups, normal - shape = (n_frames, n_sources) + +def fit_transient_traces( + residual, + designs, + motion_indices, + valid, + analyze_hz, + *, + options=None, + kernel=None, +): + """Fit one trial using fixed motion-specific (superpixel, source) designs. + + Residual is (superpixels, frames). The complete nominal output grid must + be retained, in chronological order: never compress out gaps or group + disjoint motion frames before convolution. Nonfinite observations reject + the entire frame, as in QR. ``valid`` also excludes unmatched motion/z. + Public dF is NaN there, while model_trace contains latent gap predictions. + + Low-level nonconvergence returns diagnostics and warns. Production callers + must check ``metadata['converged']`` before saving a result. + """ + options = TransientOptions() if options is None else options + if not isinstance(options, TransientOptions): + raise TypeError("options must be TransientOptions or None") + op = TransientOperator(analyze_hz, options.tau_s, kernel) + shape, valid, groups, normal = _prepare_transient_system( + residual, designs, motion_indices, valid + ) rhs_trace = np.zeros(shape) for motion, (_, rhs) in normal.items(): rhs_trace[groups[motion]] = rhs.T event_rhs = op.adjoint(rhs_trace) beta_scale = float(max(0.0, np.max(event_rhs[1:], initial=0.0))) - penalty = float(options.beta if options.beta is not None - else options.beta_fraction * beta_scale) + penalty = float( + options.beta + if options.beta is not None + else options.beta_fraction * beta_scale + ) diagonals = np.concatenate([np.diag(g) for g, _ in normal.values()]) positive = diagonals[diagonals > 0] if not positive.size: raise ValueError("No observable profile support") gram_scale = float(np.max(positive)) amplitude_scale = max(float(np.max(np.abs(rhs_trace))) / gram_scale, 1e-12) - objective_scale = gram_scale * amplitude_scale ** 2 + objective_scale = gram_scale * amplitude_scale**2 def objective(flat, event_penalty=penalty): """Evaluate the scaled objective and its analytic event gradient.""" @@ -182,24 +210,35 @@ def objective(flat, event_penalty=penalty): ) optimizer_options = { - "maxiter": options.max_iter, "ftol": options.tolerance, - "gtol": options.tolerance, "maxls": 40, "maxcor": 10, + "maxiter": options.max_iter, + "ftol": options.tolerance, + "gtol": options.tolerance, + "maxls": 40, + "maxcor": 10, } result = minimize( - objective, np.zeros(np.prod(shape)), method="L-BFGS-B", jac=True, - bounds=[(0.0, None)] * int(np.prod(shape)), options=optimizer_options, + objective, + np.zeros(np.prod(shape)), + method="L-BFGS-B", + jac=True, + bounds=[(0.0, None)] * int(np.prod(shape)), + options=optimizer_options, ) selection = { - "converged": bool(result.success), "iterations": int(result.nit), + "converged": bool(result.success), + "iterations": int(result.nit), "message": str(result.message), "objective_without_data_constant": float(result.fun * objective_scale), } debias_info = { - "requested": options.debias, "performed": False, + "requested": options.debias, + "performed": False, "event_threshold": float(options.event_threshold), } if options.debias and result.success: - active = result.x.reshape(shape) * amplitude_scale > options.event_threshold + active = ( + result.x.reshape(shape) * amplitude_scale > options.event_threshold + ) active[0] = True free = np.flatnonzero(active.ravel()) debias_info["retained_events"] = int(active[1:].sum()) @@ -212,16 +251,22 @@ def restricted_objective(reduced): return value, gradient[free] refit = minimize( - restricted_objective, result.x[free], method="L-BFGS-B", jac=True, - bounds=[(0.0, None)] * len(free), options=optimizer_options, + restricted_objective, + result.x[free], + method="L-BFGS-B", + jac=True, + bounds=[(0.0, None)] * len(free), + options=optimizer_options, ) full = np.zeros(np.prod(shape)) full[free] = refit.x refit.x = full result = refit debias_info.update( - performed=True, converged=bool(refit.success), - iterations=int(refit.nit), message=str(refit.message), + performed=True, + converged=bool(refit.success), + iterations=int(refit.nit), + message=str(refit.message), ) elif options.debias: debias_info["skip_reason"] = "Support-selection fit did not converge" @@ -233,22 +278,48 @@ def restricted_objective(reduced): initial = events[0].copy() events[0] = 0 converged = bool( - result.success and np.isfinite(result.fun) + result.success + and np.isfinite(result.fun) and np.all(np.isfinite(model_trace)) ) if not converged: warnings.warn( f"Transient solver did not converge: {result.message}", - RuntimeWarning, stacklevel=2, + RuntimeWarning, + stacklevel=2, ) - return TransientResult(phi, { - "method": "transient", "tau_s": options.tau_s, "beta": penalty, - "beta_fraction": options.beta_fraction if options.beta is None else None, - "beta_scale": beta_scale, "converged": converged, - "iterations": int(result.nit), "message": str(result.message), - "kernel": "exponential" if kernel is None else np.asarray(kernel).tolist(), - "sampling_model": "nominal output grid; acquisition averaging not modeled exactly", - "selection": selection, "debias": debias_info, - "final_event_penalty": 0.0 if debias_info["performed"] else penalty, - "objective_without_data_constant": float(result.fun * objective_scale), - }, events, initial, model_trace) \ No newline at end of file + return TransientResult( + phi, + { + "method": "transient", + "tau_s": options.tau_s, + "beta": penalty, + "beta_fraction": ( + options.beta_fraction if options.beta is None else None + ), + "beta_scale": beta_scale, + "converged": converged, + "iterations": int(result.nit), + "message": str(result.message), + "kernel": ( + "exponential" + if kernel is None + else np.asarray(kernel).tolist() + ), + "sampling_model": ( + "nominal output grid; acquisition averaging not modeled " + "exactly" + ), + "selection": selection, + "debias": debias_info, + "final_event_penalty": ( + 0.0 if debias_info["performed"] else penalty + ), + "objective_without_data_constant": float( + result.fun * objective_scale + ), + }, + events, + initial, + model_trace, + ) diff --git a/src/giant_python/extraction/band/workflow.py b/src/giant_python/extraction/band/workflow.py index c8daae7..1c0f8f0 100644 --- a/src/giant_python/extraction/band/workflow.py +++ b/src/giant_python/extraction/band/workflow.py @@ -325,7 +325,8 @@ def _read(trial_ix): execution=execution, exclusion_mask=( np.logical_or.reduce(soma_masks) - if soma_masks is not None and len(soma_masks) > 0 else None + if soma_masks is not None and len(soma_masks) > 0 + else None ), ) @@ -578,7 +579,9 @@ def _localize( ref_d=ref_d, ) source_seeds = get_act_im_peaks( - act_im, peak_th=params.peakth, buffer_size=params.peak_buffer, + act_im, + peak_th=params.peakth, + buffer_size=params.peak_buffer, exclusion_mask=exclusion_mask, ) act_im_peaks = source_seeds.copy() diff --git a/src/giant_python/gui/parameters_gui.py b/src/giant_python/gui/parameters_gui.py index fe1b3c9..1fc463b 100644 --- a/src/giant_python/gui/parameters_gui.py +++ b/src/giant_python/gui/parameters_gui.py @@ -34,9 +34,7 @@ def run_parameter_gui( *, execution: ExecutionInput = None, annotations: AnnotationInput = None, -) -> Optional[ - tuple[BandSiloParams, ExecutionOptions, AnnotationOptions] -]: +) -> Optional[tuple[BandSiloParams, ExecutionOptions, AnnotationOptions]]: """Return split option objects, or None on cancellation, without mutation. Retains the original form and natural-log sparse-factor control. Toolkit @@ -111,10 +109,14 @@ def on_ok(): "peak_buffer": int(entries["peak_buffer"].get()), "z_tol": float(entries["z_tol"].get()), "sparse_fac": float(np.exp(float(sparse_var.get()))), - "trace_beta_fraction": float(entries["trace_beta_fraction"].get()), + "trace_beta_fraction": float( + entries["trace_beta_fraction"].get() + ), "trace_max_iter": int(entries["trace_max_iter"].get()), "trace_tolerance": float(entries["trace_tolerance"].get()), - "trace_event_threshold": float(entries["trace_event_threshold"].get()), + "trace_event_threshold": float( + entries["trace_event_threshold"].get() + ), "trace_debias": bool(debias_var.get()), } result["options"] = ( diff --git a/src/giant_python/gui/roi_editor.py b/src/giant_python/gui/roi_editor.py index c56d442..abd5995 100644 --- a/src/giant_python/gui/roi_editor.py +++ b/src/giant_python/gui/roi_editor.py @@ -37,9 +37,7 @@ def _ask_roi_label( return text.strip() -def run_user_roi_selection( - user_roi_geo: dict, n_dmds: int -) -> dict: +def run_user_roi_selection(user_roi_geo: dict, n_dmds: int) -> dict: """Select ROIs on reference images and return per-DMD selections.""" import cv2 @@ -141,9 +139,7 @@ def _select_rois_for_dmd(cv2, dmd_ix: int, g: dict): return roi_masks, roi_records -def _compose_display( - cv2, im8, sp_plane, roi_plane -): +def _compose_display(cv2, im8, sp_plane, roi_plane): """Overlay the superpixel mask in green and ROI union in red.""" disp = cv2.cvtColor(im8, cv2.COLOR_GRAY2BGR) if np.any(sp_plane): diff --git a/src/giant_python/io/experiment_summary.py b/src/giant_python/io/experiment_summary.py index cb3a924..359197d 100644 --- a/src/giant_python/io/experiment_summary.py +++ b/src/giant_python/io/experiment_summary.py @@ -43,7 +43,8 @@ def _decode_path(assembled: dict, path_index: int) -> PathSummary: df_ls=assembled["dF_ls"][j], df_denoised=( assembled["dF_denoised"][j] - if "dF_denoised" in assembled else None + if "dF_denoised" in assembled + else None ), events=assembled["events"][j] if "events" in assembled else None, f0=assembled["F0"][j], @@ -86,7 +87,9 @@ def _decode_path(assembled: dict, path_index: int) -> PathSummary: "annotation_enabled", rois is not None ), _empty_source_arrays=( - {} if sources else { + {} + if sources + else { key: assembled[key] for key in _SOURCE_KEYS if key in assembled } ), @@ -131,10 +134,12 @@ def _stack_optional_arrays( arrays: Iterable[np.ndarray | None], reference: np.ndarray, field: str ) -> np.ndarray: """Check each optional entry against its corresponding required trace.""" - return np.stack([ - _optional_array(array, reference[index], f"{field}[{index}]") - for index, array in enumerate(arrays) - ]) + return np.stack( + [ + _optional_array(array, reference[index], f"{field}[{index}]") + for index, array in enumerate(arrays) + ] + ) def _empty_geometry( @@ -182,12 +187,15 @@ def _encode_path(path: PathSummary) -> dict: "F0": _stack_required_arrays( (s.f0 for s in path.sources), "Source.f0" ), - "SNR": np.asarray([ - np.nan if s.snr is None else s.snr for s in path.sources - ]).reshape(-1, 1), + "SNR": np.asarray( + [np.nan if s.snr is None else s.snr for s in path.sources] + ).reshape(-1, 1), } ) - for key, field in (("dF_denoised", "df_denoised"), ("events", "events")): + for key, field in ( + ("dF_denoised", "df_denoised"), + ("events", "events"), + ): out[key] = _stack_optional_arrays( (getattr(s, field) for s in path.sources), out["dF_ls"], @@ -197,20 +205,20 @@ def _encode_path(path: PathSummary) -> dict: out.update(path._empty_source_arrays) else: spatial_shape, global_f = _empty_geometry(path, "sources") - out.update({ - "profiles": np.empty((0, *spatial_shape), dtype=np.float32), - "coords": np.empty((0, 3), dtype=np.int32), - "dF_ls": np.empty((0, *global_f.shape), dtype=global_f.dtype), - "F0": np.empty((0, *global_f.shape), dtype=global_f.dtype), - "SNR": np.empty((0, 1), dtype=np.float32), - }) + out.update( + { + "profiles": np.empty((0, *spatial_shape), dtype=np.float32), + "coords": np.empty((0, 3), dtype=np.int32), + "dF_ls": np.empty((0, *global_f.shape), dtype=global_f.dtype), + "F0": np.empty((0, *global_f.shape), dtype=global_f.dtype), + "SNR": np.empty((0, 1), dtype=np.float32), + } + ) if not path.sources: for key in ("dF_denoised", "events"): out[key] = _optional_array(out.get(key), out["dF_ls"], key) if not path.annotation_enabled and path.user_rois: - raise ValueError( - "Set annotation_enabled=True to serialize user ROIs" - ) + raise ValueError("Set annotation_enabled=True to serialize user ROIs") if path.user_rois: out["user_rois"] = { "mask": _stack_required_arrays( @@ -326,10 +334,11 @@ def read_summary(path: Union[str, Path]) -> ExperimentSummary: with h5py.File(path, "r") as f: annotation_enabled = { name: ( - bool(f[name]["user_rois"].attrs.get( - "annotation_enabled", True - )) - if "user_rois" in f[name] else False + bool( + f[name]["user_rois"].attrs.get("annotation_enabled", True) + ) + if "user_rois" in f[name] + else False ) for name in names } diff --git a/src/giant_python/models/experiment_summary.py b/src/giant_python/models/experiment_summary.py index 6ca4ddb..b751c54 100644 --- a/src/giant_python/models/experiment_summary.py +++ b/src/giant_python/models/experiment_summary.py @@ -25,7 +25,8 @@ class Source: Least-squares dF after baseline re-estimation, the fitted convolution trace (without another baseline subtraction), and nonnegative event amplitudes, shaped ``(channels, frames)``. Missing observations and - unprocessed channels are NaN. Events exclude each trial's initial state. + unprocessed channels are NaN. Events exclude each trial's initial + state. f0 : ndarray or None Per-channel baseline. snr : float or None @@ -145,9 +146,9 @@ class PathSummary: Arrays remain NumPy-backed in memory; Source/UserRoi entries may be views of assembled arrays. ``annotation_enabled`` distinguishes disabled from - enabled-but-empty annotations (saved as an ROI group attribute). Private empty-array templates - retain dimensions and dtypes that cannot be inferred from empty lists; - they never duplicate populated source or ROI tensors. + enabled-but-empty annotations (saved as an ROI group attribute). Private + empty-array templates retain dimensions and dtypes that cannot be inferred + from empty lists; they never duplicate populated source or ROI tensors. """ path_index: int = 0 diff --git a/src/giant_python/models/params.py b/src/giant_python/models/params.py index c7d467a..b145489 100644 --- a/src/giant_python/models/params.py +++ b/src/giant_python/models/params.py @@ -30,8 +30,10 @@ def _positive(name: str, value: object, *, integer: bool = False) -> None: def _nonnegative(name: str, value: object) -> None: """Reject nonfinite, negative, boolean, or nonnumeric values.""" if ( - isinstance(value, bool) or not isinstance(value, Real) - or not math.isfinite(value) or value < 0 + isinstance(value, bool) + or not isinstance(value, Real) + or not math.isfinite(value) + or value < 0 ): raise ValueError(f"{name} must be a finite nonnegative number")