From e06d91602334dce9a3a7f5c7b888534d27a5429e Mon Sep 17 00:00:00 2001 From: kormann <49917710+DKormann@users.noreply.github.com> Date: Fri, 23 Jan 2026 20:23:01 +0100 Subject: [PATCH 1/4] fix cross reducer call TS reference (#4104) the TS reference was slightly missing. you need assign a variable to the reducer to call it cross context # Description of Changes slight reference fix Signed-off-by: kormann <49917710+DKormann@users.noreply.github.com> --- .../00100-intro/00100-getting-started/00400-key-architecture.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/00100-intro/00100-getting-started/00400-key-architecture.md b/docs/docs/00100-intro/00100-getting-started/00400-key-architecture.md index 46d77df1ba5..d4fa03561bd 100644 --- a/docs/docs/00100-intro/00100-getting-started/00400-key-architecture.md +++ b/docs/docs/00100-intro/00100-getting-started/00400-key-architecture.md @@ -185,7 +185,7 @@ spacetimedb.reducer('hello', (ctx) => { } }); -spacetimedb.reducer('world', (ctx) => { +const world = spacetimedb.reducer('world', (ctx) => { clearAllTables(ctx); // ... }); From 0f0a867d90824a3ec59e65b09256fa6a4d470df8 Mon Sep 17 00:00:00 2001 From: Ryan Date: Fri, 23 Jan 2026 13:28:24 -0800 Subject: [PATCH 2/4] Removing1.5.0 DLLs (#4100) # Description of Changes Removes 1.5.0 DLLs from repo # API and ABI breaking changes Unity will complain locally until you regenerate the DLLs # Expected complexity level and risk 1 - Trivial # Testing - [x] Confirm CI pass - [X] Local tests are passing --------- Signed-off-by: Ryan Co-authored-by: Zeke Foppa <196249+bfops@users.noreply.github.com> Co-authored-by: John Detter <4099508+jdetter@users.noreply.github.com> Co-authored-by: clockwork-labs-bot --- .github/workflows/ci.yml | 8 +- sdks/csharp/.gitignore | 5 ++ .../packages/spacetimedb.bsatn.runtime.meta | 8 -- .../spacetimedb.bsatn.runtime/1.5.0.meta | 8 -- .../1.5.0/analyzers.meta | 8 -- .../1.5.0/analyzers/dotnet.meta | 8 -- .../1.5.0/analyzers/dotnet/cs.meta | 8 -- .../dotnet/cs/SpacetimeDB.BSATN.Codegen.dll | Bin 75264 -> 0 bytes .../cs/SpacetimeDB.BSATN.Codegen.dll.meta | 71 ------------------ .../spacetimedb.bsatn.runtime/1.5.0/lib.meta | 8 -- .../1.5.0/lib/netstandard2.1.meta | 8 -- .../SpacetimeDB.BSATN.Runtime.dll | Bin 68608 -> 0 bytes .../SpacetimeDB.BSATN.Runtime.dll.meta | 33 -------- 13 files changed, 9 insertions(+), 164 deletions(-) delete mode 100644 sdks/csharp/packages/spacetimedb.bsatn.runtime.meta delete mode 100644 sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0.meta delete mode 100644 sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers.meta delete mode 100644 sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers/dotnet.meta delete mode 100644 sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers/dotnet/cs.meta delete mode 100755 sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers/dotnet/cs/SpacetimeDB.BSATN.Codegen.dll delete mode 100644 sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers/dotnet/cs/SpacetimeDB.BSATN.Codegen.dll.meta delete mode 100644 sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/lib.meta delete mode 100644 sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/lib/netstandard2.1.meta delete mode 100755 sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/lib/netstandard2.1/SpacetimeDB.BSATN.Runtime.dll delete mode 100644 sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/lib/netstandard2.1/SpacetimeDB.BSATN.Runtime.dll.meta diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 349624688f2..f70b677ac0c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -608,6 +608,9 @@ jobs: exit 1 } + - name: Hydrate Unity SDK DLLs + run: cargo ci dlls + - name: Check Unity meta files uses: DeNA/unity-meta-check@v3 with: @@ -630,10 +633,7 @@ jobs: - name: Patch com.clockworklabs.spacetimedbsdk dependency in manifest.json working-directory: demo/Blackholio/client-unity/Packages run: | - # Replace the com.clockworklabs.spacetimedbsdk dependency with the current branch. - # Note: Pointing to a local directory does not work, because our earlier steps nuke our meta files, which then causes Unity to not properly respect the DLLs (e.g. - # codegen does not work properly). - yq e -i '.dependencies["com.clockworklabs.spacetimedbsdk"] = "https://github.com/clockworklabs/SpacetimeDB.git?path=sdks/csharp#${{ github.head_ref }}"' manifest.json + yq e -i '.dependencies["com.clockworklabs.spacetimedbsdk"] = "file:../../../../sdks/csharp"' manifest.json cat manifest.json - uses: actions/cache@v3 diff --git a/sdks/csharp/.gitignore b/sdks/csharp/.gitignore index 42502a20a58..249ecdec52d 100644 --- a/sdks/csharp/.gitignore +++ b/sdks/csharp/.gitignore @@ -76,3 +76,8 @@ obj~ /nuget.config /nuget.config.meta .idea/ + +# Hydrated SDK DLLs (produced by `cargo ci dlls`) +/packages/ +/packages.meta +!/packages/.gitignore diff --git a/sdks/csharp/packages/spacetimedb.bsatn.runtime.meta b/sdks/csharp/packages/spacetimedb.bsatn.runtime.meta deleted file mode 100644 index 6dc8b62571c..00000000000 --- a/sdks/csharp/packages/spacetimedb.bsatn.runtime.meta +++ /dev/null @@ -1,8 +0,0 @@ -fileFormatVersion: 2 -guid: 8865417631feca343997ae22b3320e75 -folderAsset: yes -DefaultImporter: - externalObjects: {} - userData: - assetBundleName: - assetBundleVariant: diff --git a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0.meta b/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0.meta deleted file mode 100644 index a0714721e1a..00000000000 --- a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0.meta +++ /dev/null @@ -1,8 +0,0 @@ -fileFormatVersion: 2 -guid: ae5666f23a6d73c43b030a1b9ba5916b -folderAsset: yes -DefaultImporter: - externalObjects: {} - userData: - assetBundleName: - assetBundleVariant: diff --git a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers.meta b/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers.meta deleted file mode 100644 index aff342ea1f5..00000000000 --- a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers.meta +++ /dev/null @@ -1,8 +0,0 @@ -fileFormatVersion: 2 -guid: f112b25d371a72548b6b4959e104c4c3 -folderAsset: yes -DefaultImporter: - externalObjects: {} - userData: - assetBundleName: - assetBundleVariant: diff --git a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers/dotnet.meta b/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers/dotnet.meta deleted file mode 100644 index f2031f11a56..00000000000 --- a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers/dotnet.meta +++ /dev/null @@ -1,8 +0,0 @@ -fileFormatVersion: 2 -guid: 8554a9e409be36a42bdb316f4677fabf -folderAsset: yes -DefaultImporter: - externalObjects: {} - userData: - assetBundleName: - assetBundleVariant: diff --git a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers/dotnet/cs.meta b/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers/dotnet/cs.meta deleted file mode 100644 index a387186fa76..00000000000 --- a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers/dotnet/cs.meta +++ /dev/null @@ -1,8 +0,0 @@ -fileFormatVersion: 2 -guid: a24e03405bfe85e4996b2524189ac24a -folderAsset: yes -DefaultImporter: - externalObjects: {} - userData: - assetBundleName: - assetBundleVariant: diff --git a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers/dotnet/cs/SpacetimeDB.BSATN.Codegen.dll b/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers/dotnet/cs/SpacetimeDB.BSATN.Codegen.dll deleted file mode 100755 index 1db1c1f2115bc79e9596ada9efe982f13af94caa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 75264 zcmeFad3;nw);C^tZ};s^FG)H{r?amFnry5hB4Wa76qH?1kc1E*3TeC@R0xT|paSB! zfViOskY`+QM%3|fVa6c14vskDxQseM9cOTG+?`P|zTb1|cBd2KjPrh;_xb(vi}tx^ zt5a2{PMuq~Zgn%|yz7N7gs|cB!w*6{hAaJ5F#Ok`72^2ZC*s8;aWCaRCMUm?KXYk) zIB$g+T4L5L&#SFzXb3gtEv(BkS2pC;H{^{!XGY%g(4x9>uh-SjGCg&I5R;`YTrIyP zbNtsS3i8}?un<{N2&dv*G75Me@CsaoNMu}7?Is4bpZ{7R2cQ0QQGF$;(*NC8izF8Q z@?rNJLSXmHs1agVf7McWfPOhrikT7KPr2qro6g!!&}YPyD{rj3tP%8@473+A?yJ@A zh#J_{oOs1@DwQwTWOr8FK)&_*@`t?T?-;ePHGs@Vif22)pe{EDW)mL#BRo7#aXB|k*(B0 zPK7KkR@Y+dn%ZS67RiHk5kT+JaGw&pv|v1HDm4r}FlUa-C|<3vbije7DN3%dB8p9S z$>P-FF%~<*$x$?U$xSU5s6&bz9s|^&j61ZTpM+Y_i;F8s1k=g0q^k*_+#VB+TG6eM z0gYg$8h|D{c?ne^WNc`H-b@V5*32}@lbtcR!pA;+C0wVK4%c_ll`2a`rS2rDF%tBF zDoq6&j!>94FhQ!!U?h34T@1inpqywTiZI8{rOBf;AsN^L$cfE_7@L+tCs;s6Ip0*E z4DydJjLKXt!$h>N2=xQr-!QA9xMqarL$5T>$ju8Cu{5{YLP9iqi$4tZ$3-`%Ma$@_ z1tGM`)n7L;Q7yv|%!M%}2o#rVdJsY*6`A>LO)vLqrA|!`Ed&dhEPz(iJfDqr(q9nW zcWEJzMhG2u zpg{}t((_b}I@cRSZgiDSim6jqH6!1;N(Whds#NDe3Nb$r?5h1V5RfoLMM}&-DfYI>a&MNf=C-S z(PvK$odR?vIw_M4pHVu@XM{!(ryd-)qqNXx4=5GHlJq!BlZ-wfDQ=wU!YOX~+ zs%62^;Dk>DSdF1SAKkGz^bmC`J&=+TL{OK6PlvR*0zh-xLzTD&u{0?OjU)DW0P4RX zP#rB%ec=gUxeWAwA`nft*^Mq^643A&0KqDN;)s46>7NM>>z@V0(w_{p4}BNuPl@QC z-K~F4-};!2zVOtDK4wV7{`8*uH)>7Xza|W@Ce zi@)1=o#`{bheORcDOQA`YP2FwZ7l@MGVmy!WU~Lwb3g_$g2^NqbP8QQXF$4ipHVyAX2u`B3E#p(libsVf5x7x_~sImC>|! z?Fd#gtN{#Mi>N~F#*Uzlv28)yY+c4g+=;`r5Yr@awPH3|z4{fk5v~5|y~@@8b+6K` z?i+_N|6R&4lV7YC_MRZi>4WeF6^V|MJE>h5uYIpB0if$ zK3j&8;pG6W$TXJEo*)lkma9=T`|1BS63U_S(<0#~`lsQ<4pDA!9U4eOxdIr6B1cDW z|Mnd$_U?|KK^?ysHo_)AD`t|_@jp>FkkqV>PqY2oj@LrvUv)eX36>uj4x_tORn#h^ zRadx?T7_h6wW_}zbs)uBt-1{4znPfpsKmdRn9)`pT8XN9h3k*B6gP}W;G9Vjsf!0~1z>6eCz#vs-FruE5kaT;jf$WaRQl-}#XblSgi!~5EH{q5c zusHWw%c1P3Dcr7`QG63Tni*LW4T+VuK+4Zq=v^i+s?<76t&lE?fJPQY>zGab4_!%YUj2~QLTpj3w{UyoKdgTy>7`?}5E<7Qs`byH zV7Gal{T#XqDy4y7w|$-69l9F)$U0{|;Vsd14h@SMMWxF_S-jYC)>?8F);Hvuwd58o z5SuVtY`QJ90ld&gvUa9MkA@}9B(S0Ys)KD}JbDmM_g*;9aOp;96V%=BW_yel1hP$Y zhOZ?fxSf|I$u9q?DD|#z7+gc>XAp(2BXB)Hu}ma)7{Zk(vf(Q^LJQphzE1lECB@{k zc<}->JgCo{fN?cv0q}j;yd2n}QLz4#@4nAaiTcQ7qxD_SyKna-zY!v3`+Q)0_x%s< zvS@8d3+~6b%brbBY%^?xw*a(~gNjEbh~Q12_}RDwYYm$K$$tq_YuO4>QCXmW%gsQ# zj9UQ1w-T?FB=nG7(sDb|-8$Py=ML5(NiYFATFYHMmD)*ZJ1dbS2-lai?Chy?H|gwR z9g@(-M+v6{tRUz?3hs^sL-`?$PxpfqDR2g01Sx@3nZd}I7Q}(AsLbeZ7COZW6h$oA zMXD46jrHllL2wj2G7J~@yMz0iMbL7;JHTUW8ANsJ&hXFCpimMTWL^xJVg3@3w>RAH z3Vr2t3~Si{edPpPdF|}N+W8(7@s{N3E9nkJtMameBQY4laD+PmlkFqnQczl)Er=*K z!mlcBE4Wy;-%B-S1DFp}T`=M*Erd2i_W@6KCOK1s_XD#^|CLHRO3ESM4Q^SSRZmF< zI1hl6SxziqMu`K+ZLIHeB-=}T<~P(S-qcbncAp(il38d8o;>0bMci3%H{N3 zixrUKh8TU$%#Qn!9E;iZQ=T)+qUDh*kYKFGN!>gxRCqbS1+GoESNQIusi6CV5p zb>N}QUIn4hv*6lpW#rTo<7PYLeeO^Eo46rcRTQH5!-fNEdvtS8)SPbqcQ>vDx4|os z+w^m>f;*!E6;jASy+aDKQH9hWv>+z2(PJ}$JlaPfkzZp1iAIBFwLwk~{gQmA?#mV2 z1L{NhDD;D3ExP4Eru8PSphp;yZJmdZw<039V&ns(_zD<7`Zf?lR8R$nQsxNZc-jU9 ziYMy(G>)M$&_xWLP8U5IL#44XL@FFIj396{r|`40Nx>-CnSGp!$j-@@OA1M1GBlAsUSk+xZV-h)nBETtSZ* zBHKC-VUHO44K*_+hNxgO0mKs(;RxXvB4+Y3ZPDo2$S9eGiF`kWdm1C+<%Z)=i%5|AB!FEEFo{+FkQzl_1 z%Zy20edVtCQ2pP?T~EWsedewMQGtrP--7NHcQ1qWqj6UO@AnaRG!`*&MUr~EIhn2dS{x{<)5x&BBHX=L6y5Z)=A@&fp>Xw`9rWp2^U$9=Tmt^r~f}7$# zH-Th{<^VSF@#y3EkY33}F zsaqh&*@W79<*|u$u{TemOZQR?4efs9B~8v!j38}M8D=m729;(8Bk%;} zv2B=lNs)QiGmqVm(T(M}&3(N%o<4G&H@B?UWaK;a|7@;%9Sy;Rv2xwViu#Xpojabb zSh+4+QIsQ)y3~VCE-f99x+w1f$)(wyXO2^v`81ykBASIXpZi5Li%7G-S*tV$Xu-c@ zXdTYD*!&obgx@Yr#RNrx`833bFU+`SW~8l)Zx)0>;yYm9y@ zVWNvaJz-k@v=XM}&jj*kV#G^6@@G=SOV<6!@{-^3Qd~?bO^u||069A?l1kIbV;PZD zN_WkeAtTM4*i_orI_)}m?SCVWeuQ!FGmjpN5%kET6e+QJ^efQ4^5{2U{j@wv$)HEV zRGSWz5_+&zQlrEWlSe7Pgt)Gso=2%dy@{*skw>XUopVypJW5sSoSJ&%(eJ2-F?o~> zHWR?wk4&kIs0wmgU~_02(a?gwf=Y8W z401HGho)wO32-P zL?SsoCK73=LTs_mJvTC~hY0rEb0gz=6Ia2(7@#LptQW|-&ag+&B~X*Q_ui;nGXcaL zmEh>%=MC};#TmH)dvAvj#MbL59{+>L)hRUokE1r1VQYSZY7J7@8Cvs`xPNctl2T1i5O^h&&rCaN!Vkl1Cm{wXD{nBC^@G4QRH&H{n-x`G8 z1*tWWYXcSkfS==qZS}UB;iF67B?`7fmX2sQZ{%4_)(h0H3Yp2oz#Q{XHu5 zKJi*fLTAHNQS%Gv^Lq=I`DG%?Q-vsx)pKn5KX;cWNlf`PpH<#%jsO}d|6$Maf1&b} z?uuGT(w)Q1Ig#?7kuOOh!heN0*=~7iFNEwF<&EgZGG3u#ITnaic9aZvhxkVzt;$H! zT^a2|uj8WWk)nU=Srj*pNKul+7R@ky$*7ed&Mb$23P!SFeoYsLYGQ6&WMOrjw2K02 zpsD6XDa1(kVP%+M5t+O<4R?Rsvl+N=MVdj9*k)u|MYHMIGRIs>Mq}I50EWgT(hoI8SFq9JOKv^6v@gV#W7R1F2#S0Iyd@|@?r5^jXB%wc|!DOQx&`HdjI z`AUxSIa|JldQ5C#YDPweB+dT>utq*5GV=d`9dyu#HENo9Ru+%C)eNhqOTn~iqF_+F z1(^}gJG%@lx>PmYo~MQQU)Wf-8JErecQ#S=%KwF6vHvM5%0OTd9OILo_0vmF)K49T zn%g`thY=4oIoE4mlgng0lL_V>icDnE7m+40nQTtXhjL5=#`S&X^tS~+vflm{UHBL4 z5?bT@0{Ji%Io3LF@kP{>uCgxDL09TzS|T8GTjQ>q_QFVwP@`-cS;-W}3MdRBg41IK zl!|c_zblnuu7}|nv1*i>JxN+u>NMo}7&WyA@*SkT) zU7%GbLs)b~k9QO^n>$r<0-L)O%f^9W>)sN=iY019V>Wdlgw?CHcS1{<5Vuk7(cG?@ zhdrZHZv?8Jg4+`OLG9ETsukCz!S0sod@BsuV6$|A^_G*Gs;f~L1T=cGmfEWGTvU$* zgM!_S)%kW9LXx1jrQNO7`NkT;#J3ztF@k*>{!_~=tk`JUtJ}Y-SKawlUFxQ*{n5MuSq9xt+P`7MIt>SF5O41>WZPKx`}#*l-2Qed-(;d}#UIZoU^ zd<>=geD1~b!M~b^cmfQP3LoX}59__cx|k}Yque;~j=fwLzbdEJg;)}OfW!;Ua#};V zNQbn;rBss4yNN|-+>sLKyFc`u77uus-sT%5$EEIfpi%wP%w;#0xT*>(h}|| zzL>=EEG9{?5NjSi*bmV36d_p_0qftpipHKc@J*jc6(ng{*;5=Qv5%`D3FRkLF@UNV zOH~AcwOrD(Oe2*^;xZ%&4kYarq>V|9()dmizD;F4qz=_urb8)|4E^FAbRK7I#M(*Wu_NvpMxAM&(e$yx{pTwI|5KyF@_qybEYzjbjZ?u%Hz z<2?CLm@CcHQbTD_N)+Z}XqTq(IT-bQ0*#NhxQw#?8n5)-j&@sdl;s}#~;KBSO3^Ie6^GhG9y zY{?9BghI|V7b>LE{Fy>VnolTXkoloP`a6Q?9FFjmWSMC}@)>t?(+sS6*m03lG~SXC zPBj9q@?`S@s5+h9?!`TM2UuPlze8H>PNhtaSakJwgfgK}_WHXnN2%cm;TxW=w>8+K^KP1m6J;ZW z3uck`bg>jyINOI)VcF2j7&coE;j52$j0Gnb2yS-4JY0kM0Pa*T>hG4s1q;A6f+xWl zaiK!sae+DBxNZe5I`kz+F&PKThVl_bHz=uw%^v<>VTJ?08#(6P4e zXU<|ieWK2kxgDqA-XR>pO4#zo<$C8%Ffb~}k>X^0{)7*O2wktoCoZ?3e9g$$fH`tK z!LcYq58}C1<%0lcl}{M?1c}Uc)XG~2Wy?n*CxEjz%BkhkZVvOUC}*z2JRT+Hm8^D{ zImMCk6C7qmlsKd00!Q~DxU$NUa~$TvsB~0Gg}EM-r(}uwC|?(tAMo{DGoys~=a^IZ zI>~I|>*?lhzMgEp#n(X&Gqn`A$#Ljcsn6^W+!1pYPUDROa}LEU~njyR{H(5iR1fR&Xdq>th%LHF^&M!xv!3 znio=SmjHy%fZf$2w9JKCZswwDZH``pGhq*rq>3bw8w%#QM*9^an_`W&Qn8FX6B@Y5 z2G7DZdh0n;YdVXilfmV4uu;(kTGQDqnnEHRDs=^Mr!R@hsSe!M=)1M30}Er+aSo|Y z1;D*SODQ%&(|{YH>BOD^P+U%~FpQDCy^ro34!}UT~LCWIxr{J1P`6AQ+E?S;P zUx^FOB2|}e{a+qN>x|Ypz3k0ab)Ab#>m0Tnxz*|+BnH=};Yb6;SQprW*rTPYLl_2E zaj{y4rF7|NDU2Ac7sOirHY>7B?10Z{nMmt(ULhW2MtX#i5xj;4ndV^1Ki9(DO?>Pw`y ziTD~8n4x(9n?ls@Vm&k;1T77IZ=QkD8P;RFI^Yo7uSjj_sy2S)Mxh&H3Pn@`koTZGiblX27xJ3^QR6jwYm9ui!JxOprKO z2LJRpoY*~16IdZ;oX%xNALDcp36vMRi*M*v-12$4`EwR@4@T@sBU5$Ed=5GmS~3_} zgywVc#NJ&8X9%P*&WMg&i|VuzX}k@ihc2LzV+Ta_KD3;3T2DJ>RZm*ujf9AWYY|pa z3)8ixC($|^o}rP`U7-c2Bq~Id(p<+PL*(;Z{|!|1P&ITYpVZ(QS_pvW=pg1*Nf1l7 z63m|v%^zCCM=cfWD>7=7VOt@4*~d4^raWmLY7%H!^a#HEK8gNU|<`(+oG`sGlf(P$uL9iLswO+~x$h~?AGaMTK3Mp+%~O6At!#unWIx-wP` zfuuHcu*8)o?9?ElIPLXM5PA?X<0`&DUx`~+W~o=xcM+nvsR%YzPi|1>S_3P#6qZvEv*4uxd2uYWy#O*ZEpO^hb!CyKwrMG zQ3Ffd`pSh+;P3k8JMd-S%OO_2Xa%N+Vr{&tIYMj6#ua3v2{w3cYIN@vL7o?6I5pS? zIYrnuw8eZC-rzN#inV{Kd!(ms1?*zg8U6miC(+^Sd4@N5Eh_pqdxjJ%Sc4!|zKdGH zz->fHG_@=BXAy6!VARdzA(6m@+BH9-rgrtNgOm~t$I6W$igyIp!3X?3EL-SGpw>Ii ztC&l!3tbI#4h0qN?ivRbAwe#lTKsH`0sf|yErjJi?R#~b;BQaiOHttC!M!}Z1vR%K zM|EdW^qHQgkTD*U8Gx?RRr@WSu|mZ2n}m@Ns8p(ss|hB*js;BYff z2F6k23T~n*@dgZjca=Hy-12fgcR{g^--rqQj12KRLlvsLQ=`VeJPb#cv4u_>Mq=_;NG39?n>61oRgscwjvI&}7LWUp13qgr%!v*Jg-nL^d* zsxmw%Q_RtK_Q(tL?wdMIXDq0X(~951aIA+u#uKYjo5nvi(Y%pSgt)62i(YUU-9%_G zk4%;yLT2qg!$uZ(s2RaflPzX2l8HXy+uZ`qwsM3P+>Pq7yjVkXEVvmpr4BjCF#i}C z1@kjtN{hayXxp@kGWqEg@)ZE)SEP6oiuJdYD5Gt{?LM;w8A44PtcNgd_M)ON<_@L} zNrIJRV=O=j&*0eY>V&hMp<76y=~h5*(`|rw>035RYoacPkgDk&coKFD;6!X8U=vK$ z4bC0ll>PbLc-;3>ub}Ln1;*Qk`CX(Z*pVRDbGovApaU!pJ=*h${ANd7S+T~inL`l4 z8Cizu9cpDZ^<2tQ?tOn4;1I23Re$7%KO$`2OEO}71t<{~9-J+YZuaSb3Xd}7d^hl5TW6`{kMt%SW2tSW}71ypOIDb>e2B$`+~Qp^sx z-D);cW%Omy-OT1jBa8K7bCa3P+x|SKxs9rQx2xv$dWXdA)I-$Tp6`&TrMhab4OK~5^C=EBd3AF%k;T3t;^;VY$8T9&G$i^=YNJdvO`aapz@8sZ5bsS{k3PjE1|;paBO{Rsrm;j-5! zEKfxX6A8YT@p6Gne3Q8e7TTfd61&m}pP=XD7~)1hm1;~Qd;r5gq*F_l>Sv?uCg#)t zy2R&!P1!Cn-%aqbRD#bK1bq&ILGTSR-%j>lg47VdPHzjui)Rb3DU264_anF#&?VmF zvK@f&;vhJNn8Nn|sBg-~ufW-U<^DwcH3M^B)D)8CX7F7i7q}s2!@eOp154oHV*P!? z5RY(=s#sHU3m+&V&7TFR_KUdQ0K)^3U?&I70m9$;xz5w)QxBar?HmqJf%)RL3HF~m(g%6GWO!=HC#5`2LDa68xj z0s0az-esSZgJXz~VEH;pwHqS97Jk7aSjc1AkKv;{Ms~E*5GA=3A-8egQ+aeeTq;zw zG0P>M^y9@6uJ>xjHQ-62wCGoEmv}Fme6rg`ExaLjVum5^ zbYGs4BOh zGqpR3C`2_=uV7YAKt@B(K)J8DhD3q2C8+0E(c+mIJXa}-nWGnFLp z#0=RtOr?klRgSHsh)SmVGv%k#2heP0DpgEms+6fTaVAq9i#m&`Hm1_)+v6y=DVb`| zpx;CVbxMk&u$%|=E3Q3=-$9l5N!3Le4+ZAKo3ogDQYWfJEEW=)@mWNbatAIJQ*)e*Fy8zY@$vPZDK8I7@DA{D_L_KsC(dt+n5@~)EMyqQ$h(0Td4~8N?G%_94+XO2nZhTjzUkbuDWK>+^&B+E;GUBQ z#YALcDmPt}NowbP+|KD@kZc3x#Y~@%Z-t)1)F|B1h#J*HcB<5&8H8n$jb;jO-9XJ0 zW0;!G)GYiEEoovi53<=}tkm-p#h);b=8JRjO}}nx4*hN}Wcipf^YPO8oEX^!VisF@ z9J6n}s1|c#WHsVEmSKb;TP7AWRg7skUo00(nR<$AZx9zU^)<^@h-EQWE*1?eTgz51 z6)TyV$C{UkOQj2a_zW$ZC02=5QiBI;(gm>9Vm-?~^^$D0*uY-8gZsBeY>uhna*GOZ z|JuYYT<#c`yHeaPHPn!d+x>j8UfdT`?ppDL>@Ox{uTAbR7W%IQj89vioG;+eAdz6l45yWgDYRuItl*Lc^&wPjF$pRF)eO$^6b_-avNN;TaV_D&adLOLdO6< zUo2z`AG3v{PU381X)LZRD^9LUS-1^0U(6$$75N08WL#(bLB^j;C;kS;|Iv@|AuNq& z__3cjkED{d{#^T?xFrpl)W?2Vaaz8(Iql~%U(CopTN@%)W!#xOLS)461axOV2KadH zOMnYJuLGX#Is*7$_TNyp)lq577pKN8vgeCWGrq!3)Rmb(fKT@G#aDpIVy8j)t?Z>M zxb~ldA9NdU(dn&q41t269~e#i!nSz$d*8fEP1-+C}`h+!tI! z#9Mj(6aziINF%tP;Sq*EXy~1hkwP$+;fqG9*edn_UaSpvq>9D1dcZXMa}Lt!my#;J z7L{&849aS75hom%iq&Ne;0D`!fG(}TohpiL1mCt*0xyu^a};$+vP#C-8w zRug=9dDfNCS?#?R@Vj(^qTm+p<1XM^wEF;G77qiysC6*^YHuo5#!cRQ@c@sb^7$bi zyDOaa7|kk-W`A)~-fvU-i{EFx3>eORnajQoJ70^pQ>Nx&s*9`Aj^SZgia9-~>L_LS zrMN6gxj|hWrAW3lO4(s$7g3UI`XJlSvbCV>xZ59OYO8a3LBMYlhb`(t7g28$CAW%8 z3$j3cY*Fi7S$@0tfiFtycrPkxp^POP_Ncl(cFJ@E)rYlZ0|>P@DuLaWRC zE^)-7E_4xfR8i>PP*BI2Qnh=;83oi%s(qB-E7mdftf1QCMTbREeevQ4rnZXJc@_S6 z;VGnYTSX(N1d*;N=Z4ft{zOqq6ngu-e=?|>S*FG>N!%5s67ikOUnt7C#(fOm7JAg8 z&I}Os8;hDAK)w~v#gt1Cf3RemyvOh+|J|r;w%;$lv}D`kpOOJ_+@d_t3}hk%3bKs6z4}}66bkp zdr%kkpu#<|tejEI~C^g@oBR;bfJJ4ob-JViO* z$yw;n7j)E6$(Dec6s4B?3&ng(lWZ1=x|niBVtG_k;$0s7vTJ1gio`lglcJ!%*c6q` z_xBgKTQagaK`6sHr+T!R4}EyJ55YhGUo(Gl{QVxw5X|$PyExwe2W_IzrsFU)LGON|3CaQ#0sXA zKWB=)OeufP6e)wLKJw>Q`z(=XQR_jSD~2NS!ZoE$X_0+|&hP7*liylA=bO!PHjqSASV*jkqUDO-QX3 zk1?gZR41;*4i{zN5!v%o>v+G5sA*X%QRCaaxJtZhQRIn@!h;9#h$pTU zbC^0HXii@%9$<<*@iN}cK5kL#L0vCivdYn&`9`7R0iLXoCvFrzralDqWa?&-rDX8S zo2gsGQcIKka+6rW6vad*s2iA4e%UH^DvCXEi+GYL<%!$Gb4tdZxJ|rlQRIo+#2Xex zp156fS`>NWc5##`<%w;g;1tzr@FKC#=g8tJnC_j*BNp{hd|u!M(ZQ7R(u*Py?F{Qlpso1Mz@Y1<~&qUrNRUdija}i{U zymUd}b1{o4<)tq~NKug01-=$nGNtZE{}eYU8Q+ioDQ>kWx*z>h?69b>y-%lpBkr@P zAG~>iZ^Xljg3aZDZ^a8yYIWdy@itRiMQzr)zz-sD2KQYI%-$T3@(iZb{YaBrm{Rv6 zP5w@4!b^7rborJ=tp{b7f49m}?s3TSD%E%Ll0%MG6#Q~;z$tICC@CHc#K}9DBELKe z>eoz>Uw$2M$(NZ@p72Prh}mzyC1Xp1N4~8zQSNDcQ~nWboYMXM&6M-qKKVRTSBclM zKTk-Jol{AsdXyqdrxB%klp;T5ipKpaJQ#m&QR_hkq?2B5S+gTu-eOVIqjY(XqR^vT z+!wN+g1O-U=0 zCtK77d6{YDvf84~3lygfmdh=wIDSyt5V^^sM8=4;p>pILvU!8(3Y?ZUOx|Kq_JS#C zBcx|8$*_{knw@rv{FEuRMmbd`o=39HVvVdxJ5`=BPid}7pCCuexfXTEyEJXITw+lb zkd2XHrc@us$Tdv0iNo?zZH&AlMphwr#mFk;V==O^vLi+|R=yD<8!tO!WaFiOKDA7> zdZNr{O4UA54rHoL{6%X>n<$4^)T#C>(k98#7Io0RHSG*J&!SEPwinZu$!d$T+y9)_AaAg!TkT(` zt&q1erEHq=ZcFw@yOwUsK?}R<3(J$4Y7-vAogS7GqO#MCMtQcPoT)`887t*%i^?lX zNxwu^Thy=pn!Hr5v8dI7cR}646z#j+pLm&kfGKtNT_q1#vgb38cvi_jFm*)UoB0sl zWBk?9r2G3SIlPwIxt7<8tK{`eQO3MAakboLQR_jik^jZiv;4%+B7ega-GTDcFP8@u zg*;f6-YPqz)bRARa_=JcgmZQNOy?EyuNL)Z>}p;iQ|d^jviB9Tnkm(mE973Ls4b_a zx5?)%YCWhcl)bN(-&qu8@2h3%EQ+3>ZxaJFnHhSx#k2`D}~a z#+35e7WoxZEA7_fjc6Ivg z@>NBlN9)q>l&*TRxt5=A+hri&~C#>vlPk zDb%EEegx8|K*!uQ0J zg-pgrFwEz=#<7KxJkt4R3c)f*ciESi^IzN&UoP>9qY1@PxZP_Ula+#?Cdjh- zlcEAOYN8{icGZSY+5R2*B;5fhMJ)aqw^Q{uB~^p8!mmZ`tU_5$w0h{@a$J{yqg;3! zMi%~G^OK4qwcuYzg~v28CpvbK{{CmZi*0^A7 ztTuFyPJ|O1C;wlyO3fo`TVL}i1N_K5x&Sj&v*y$eko`zq zYkBUg`5VXi;Z)S9i{EjM`l)qA@&;adqjy`86C`JkNH0#T_orHp+^v*lYONIG*gk%O z34KwV3zv@MVtGR}H$K#+FLv40aGO(j&^8KLwXo zj#Zghj%!$R- z9X7Uvn%}QNN&m;bcS>mVFMeWqw zYF$BDNLiz^S{hE@^tSxdY*Hz8H~*CHePkyRITxd~H0};M+lkW?g*e|Z0Ot{k@xHr6 z6yPUT`s2NAG2YMO4K2=N4#sK9p*R^i3}?89iy=61x&M6|EJs+<=djR zKm(+w`n+ zLVJsTgQRmWmGXn4E5P5M|1-cFQf~wNjd7pe4>fMoN&$s!IP34xyD|pbD&^G$Bfue> z+a+;^%T!yX?N)ecf^9UH8V`7#?Oyp#&V#nek~~T6IbfT~(s?Xh#P~8mVXKkk|3>Dl zVa__nHvwK}+al?7-WExx^S1C?+bxpby>F59mVJw)x8z&!UXij7AY ze}wVf&}jyT;1*bZ!M<10Nq_^g(($qV1^sZw7xshf&%=;r%LDTF?vsqWfuC-C05}~# z8N4H{+BnR8KL*Z5?MuMxwWIQE_fF$DPG;=^TwuHiNc(v<#K}hO6@5{Wa2%I({_eP> zojnipzmaD1OsPhmDb>gikIS#U!<|R; z_`(~V!?~8D`bGIWQP!6BjB`9VQjFK=1mAd#PV-G>&QUou&>uLR>8sS<$r%!NRKF~K zLfkx!P7cn~=oH~RjrI)}SvD6H-41=)TU*AQc^aKAoTt%lVxy&#o6r@vK~9e!xAO~K8{}B@Xpej;cZBOGIFnqJ+U*7NU86NRBRN{5Gm@hcPxVZ4)yt$jmv@6cD|eYUMW@r9Df$BAYCzgU%hGAa^<#KcADi*t zC^f0*1MlP7R`=K5diingac`;qd}dbsaBkITeNIu8_KG-Cbb9<~{o#~}@vmrc2~Wva zQ2SZXp*FA8ZqAz)zgD}-b1vYyIn{t|IrZ^1^7E|K@#DF!&k$GF#aD4Js`OWL?gjpy z{}=Ilq!ZCQ+0wZv|JU(HF%$n3KhQ?^R)UFQrcOD08t0aIj4xtXBS+-qCM;vTUZ%S% z6IRLP?n!{{fhh??Y&+6sBxG^!oh->?hZ%1~U05N(CodZLn%fadKs16=Y5VWAOA@MV zk7un(sF7X9Ow4zxceF`CDdQd1DdWA(_}h#h)+zhF&F};69c9QWU%ocd?H5P%Gc)J< z>SasrDCaRqr8vehHc3_XaPY)QWy zcX@KW7%3(t#p637^8igDBwEL1-M=~P->GX0K8t50d5gz z0^TB)0B#cv$xcx!R{{=}s{vnO{2hix8u5oPY-YHD;cE=%aEsN+LtQ8vpFA}!{hQ$+^!*Ck*+sxsBpQMZvIK7oT4r}Yx$uq$DDy5b=walqwPMBdE z^V^u;hPPIaW?jkr&4Ax#ZUOuxWjpKav>@5s3C(x1cCqHethtY+`z=U1`=MjYeui~k zWt}4|eIG0R)yeORYXCnG7iyoR{F!yWU>zx`<+>ytU6PJnQv36nUnq;wszT-z$~IAw zUC25`%o)b8k~POc6MkUMIA~U8k7LaV%$dQkhBa$hvz9rv&|I2b%bIn}2{T;Fnr*Dv z#+){2wq&=l<~m8exRUvs8E$9&ovgo;IXk%xJ6V4ha~@{6AN*a}`bNdJ~y$kJipf0a_n{BhtX z<&4wF{y3Iaaj6-sQ^R--Yt}+Dm{ZF-bu0}tY(*PBNmKzvd&J{c^J}fGdq~SkKukuU(4Rl(pT9+C-aXmd>_)JocCEOb?&uJwd>fl`wEc8 zUDwG2dEle(tWya7%$!2zS2AZDIOsKV#_P93ze>mUNmh*o$#N}Qu4T)$(7!OJmi6n| zLMuz#EJ(Gtu}&N7?Bp7EGQOX6_OqS+;NOt5pZPC<^GV7JZ0&vU(J$ufwl?utj&9>| zv{BpQY&7D9_=%fO0I4o~b5#5(r;v39+NibTz%g>iF{he2wX9#u`mM}qV@?}$*0JSw z=HJV3KTG$sw39jSgY#|X`^=Gcszt{?wF__CsSR<=$zzz$u+Z*?PCq-fxu2c^K1-#sS&Yt-2Jv-#iyrMasQq~c`;9r{m{Y}^TISS3|GShn<9_sIJM(uke{lt{;wY<+i+fm< z<<>H%m8ETrx3P35b2=FAXZSut9Y0HeF?G?H>i9VfWIq>qGM`}~OAA?A;o=czIG&}| zjMp%}h~ZkmE3&pTyc_WB+z!^+$D9Mq>156kw)p{bgqwTnrZ(8zR4R`-6$~rE`B~O@ z<}U)Ao7>9#b&R(&-r?raWPBgnIRNQ*DV@ykWd0H63lG=j;TZN%sXXT7GpB+%70jt* zPBr5-jJGnrmhpDRw==$v;Sq)(c&J|K<-T~y8~Kb^0^gQZ!?4v$K3~iDc82>{r<3`e zUTWbH=7@N1Z9KP@@d}2O@l;DSs>IZ)d!N zVJAb8Om?anwlZvI*uk)qVMPk-Fl115s8#do&3O~1#@jS*W z7*;cEW!S;6lcAt*2_)w+tYFyAu!CV|D%tF0Tm;yDfb|(xFsFj?YR0P>Z)Lod@pgtC z3_BU-rICdShSdyP8MZSN>1>~2MLN}3!FV;})r_|?Y-dh8;~k85Fy6^{C*vZ6tud@% z*vhb-VF$xbh9Z+S8CEcCXV}58Gn4(mxX5BZWU)TO3g%QWUd?zli!LXWPE5mk%9Sl1eR^*UoHN#f?o-yKq@pi`B8Sh}&$(&BcMJ~59m)psB9^(}Z zs~NU3>|of*P~@>b!wQC-3`IWk^T}o&;}wioFka2Dl{u}9w=>?(cn8ByhN6J&Fuc2f z^ETrL81H2K1I9%mY1#@oOEX@DCcn8ByXnvK_$++l8 zHme!7GHmZhcG?;5V7!CzPKKh0eOSb8X4tQ&KrG6wV15O2su{L2JOJr;DWX5+rB;R= z3=j0D^|Khjb_TqNXVUQlh*Qn@0mg0kx2wL*9M5na!vpv?O#@j6@Z;~H+z)_{&ia7y zbpvVjvTh*d{JR0?=5`FEc6Jt1PxDF$wlZvI*uk)qp(te?hW$!ueNw^rc*d(4Z)Mod zu#;h48I`JL*v?RtQwFYJxUQUfeK*4n<{V&L3?d!dAd*%LV$U;dXZ{Dwr?0Jjl9I=; zVleA4Y-QNau!CVILotN)0Wk}PkbXPE4u+i!#ZdM$!wQD&L#h3p3@e7Qe}-{ch8+w$ z8H(XtmSM$k(yU;-n(=DJTN$=9>|of*(oV+3No;{(HN#eh>rSG3N;~5n3_BU-jUfGs z5!9m!#;X~xX1tYQJ9F9@?_j)x@lJ-~WUi56>&evS5AdrQU*oqjymGj_QqI-(XfJCY zYXkL0yUqC4QSZFOd8Koc^9|>_u1nmnyMO69==s7^<~8I089z8-Uc&VW?Fqk2=uG%$ z!qmi{B|e&%=F9Od_9Z1b@J7&y-)xEFcYZGHfVlBXEMEMVfNuli_g?VkE(1FvnfQgL zEPSsx2i#n6^YJdP5ZnRa6oV?kH&ILR3nXR0dcGk%1^XtW#F^q$ycHaSw}KUTD>x1> z9h(XcrVFvfMKHtv3gA_)*8rOe-ef+(Yg69^Y<0g6I6FY_dN09U@qYnKWW1Q+;8ZG` zpLY!KXbRzZ4$}N4kKjJ$#O0DS7FX63=be6}nVrWt#oY!1TuiP zdk}Z#po~X;puoA>_TK{hy@2=Ja`&@+s6$U0FQp}H36^? z5u)MukdqNfMS%3X&wjvx(9#ea>42q(6b-SF4V`j84R3#Q0Z&EbXyP>3&=4C%ke&{x ziLtPu;kUyE0-k}0((s#=Wq_0M=2^qLYLnQ5oB(_S{s|E&HUetc532&a z9Xov*_8_N#e+Qr@?!>S1NO2b+{X10C0DpH_B$fYvkpC+vHm4+yRJQV_yut2Gm5myb}0!Kuz2)uLj&LH-Pg1Abf)T zGAVuucqV?|ZxF^gh_M(4f6YdZ>(IiLq8Z;3y;j_WH~9C7$3%xX2yeVCQsp^viENN} z$=Bp4tya5Sds2H-`-hgK57J-LkLmfgMYbl}leTwlAK9ecYwu?tY@cMGVV`He+n4X(GZ+iab@q5GGTfFysz486yPl~@X z{^9tp__&0Wgb@i#60S~oBH?hthY9(K=OrFZ{5L!EAS2WjrLvO+v)qA?`_{l zzVCf;Noh%WNw!uxg<(eqjQl;zCvGAX{xO@RH*vazPLAqP=1EDL_`Fe1-bBn-{%z!_ z-k79c;RK4n*$fBfgcG~)PRxgX$iV&J|9;qy?}ypYAG0lpnJ^I9cra$cV9dCo7@eVF zJU$cfnTXFMoa;CPpDKLL#5_9-pUL=4!RKsz&cSCYIQaHCpY@=6&k(6N>mwlFj#Dv0 zEE2c#)wKAH*7XwWdYN@?vGmtj`d3-k_11Nxb=_oLw^;gjTG#I_J5s88b?fS|u5s42 zUQ_brxHgNg5WVHvGh(m)j5y2oj5vhP7x<*wpAo0n1G3A0Tof5XE;gPK7a1C^C&@y` zY2Z(gPdcW6J}wHKA^98US~(-`8u?^gNWK|&vrKc{AqTsTi%Qo{;Kv1hYF&He4%fqS zFHU%TC7+bS9g@)-0AB`tnP-Ari|ay}yiniDrk~_Wg zwLf|nXrFu6YCGev)!vA|PWwvUA&V29)6!g1@G)S=K>6cha3bnYJff{h49GX|`AUZ5 zcwa!DA&)DRxCaKQrn0|ukVoI=+H zz^2xi;krsoK#UXBi94ot!GiJi;T6kjR*hR$6Ar7wWQMI&HPkoO*DR}FT}L%At9J>- zqy|Cz zPO+hgrIQy784?|9V0{cN$xj?wtJL5HKza{tA0=3iLy8Lg$k19ghbfgG9Ym;=*G4)p zf|SWHdeYOa@Dua8b(!qlmyuL1>RB>|&JGz|9l?N9O=wuT9D%WLSshasudHdbhzXbB zAI}Zb#DQdNO}LJRZG2trvKqvZ;!dxtS#(arvQ^_k%a+yETFk13TC;9>T|;Bdve`Au zR@Q~5nxRYT7uA`O3YOJ3qQvqQHKxT=O+WxZV}fah%<*;MTC;wI;_w`uUKd`utP%1V zbts28UNmDBYPf7NrUR9Vw7b$YYgSQ3%a^a@jzkzs!E3BrK19?v%&swqh*?z)jl+gU z;{#Y9@nOjaF96az5+Y?u_9KICsf;9*=r9m9ab-ho^-#g?nYnTWd_PoFD*p`?XVtA@ z!PJ_16a1w^x#2@q!y~m0#*c>d)naf}S_0C$#g=Rc75$MGLlPTPOj;9jm}>Q~*j5kY zrbX-yUNF4RHi4_!1f+MHEEy8&Nqv?DH?~0_ds+a4r`}-vbVy(R0JmoWFsuUj8J#|c z9NgFn`%oH57NS;w^gchw_RyPJqn9*a-!OuO8ol!aS>;)66CQKf`V8*P8 z6RYNkOK7$%SRf|XEMK^&MpQQ92b34CM9RjL6g8k22`_|&S}|i4?gPuqyJudwoKt*# ztyogmxS%RLqY<;TR`ecdmUiEHsuqbw(c2J+v+FKv#HDi4B39(Nw@8Fx(Lyo%!gY(p z=vWhDE?Tf)Y)$P&__=4!#nydw(TV*3)85y{#&wCxKWNOeq7 zvL#!woR|{z!Pt^)k+hSht}vV#l4A{L=$WA;MpbvlTcKFg0g@js*1#4>+6I+T3$=?F zh|vVA02P`P_KyJd4`ZPp2AZM;nzmasz;>~z`#kS^&OP_eheWT7E{cW{=bn2$-}8RI z?>)S8yxgG@41cLo?a*)<>L z-~&*=7I$u5$W=Bvz(^h10rJga2@0}v`BKc7ysoTWW7xs&=R(xZ(n|@y6-G`f{OKYoBMfx(ZuVhC|af<`}%UdNo_@*ogJv zS*&O4?c*Q>)lSu7JKDR8v1qFXnpO*W8cOBv75W=ynKTn20aF|2aGj`D)_Jco=^XrSYId3x2A8a<9q8ZEHmz^^Z#+bq|M>jfdk zDIf!5ZIvP`T1T|n8f;1iku1JS2Sw{RC{(im&4yLzv=njm3^89W70U%N;UT+d>cu*` zy;Q8Poy1G+v!X$28`)f8p}vKNg{}4FO34Nk9ea>{0(B+?^>=Ke&XH;UiY_H z3Hk<#NNQk0yyi5VnC@GtCXEYgh>WD9!txyOOT;hp%DsqUl|{8gQfY~;ypu*Lux7qy z?V{8ym1~jvsmra!G*b@ zvBxjxN}G8RXR5Pf6@Uq^!cQKQmgsE+cV@1zyt%qc-tcvfTYOE+-h6}G-y)TPub!(E zYG*2SHQe(nz5%RBpCJ#r_)68gCnzxRkuLZKL}!I6v>?MQUwb;cwOk;Kz6xuxVt&OI ze2W_5w(8kZsd6bg2MC>QuMHLhKb5;o8*K_wF}}CM(g`Tm7r{U9G1?pHNYKEG1Jc&( z7+@<@o%i6wRzHvBm)dg`wR^tBEf2f9Qe55C$hN)N1*8$^B9!!F9CIpqeLXTSS;Vld z)!u`*3c-&+i0VWrulXFcT3*{54J&+z6Kb-e5V8#v`GxFCp}utvu}IA?Bzn&34eH)d zlSAF})xyPM;gS`=PzM!Fq0PBMDSKH;q4ov1EK!t%8Xfwfqw<@%x`)U$9II?>Rg0@@ zp{9ik#SId&RO7b&knY&Bex>i~q!?7MBjE83ie>d?e646>QGmW~9-GqShp+QMBe_xD z@(tEKA*z6$`RMC|@as+F24C%rC2^&_5A`np;a5svicrsTJ;{J z(8BqP3>e^$xx6L5v4H@gT~Lp)VT_~-6<=n zY*7bYGBS$ez+*6H9wxI$dZvl2eKYT?$7w$Yua|+M zW7TB)yn3ES(Q(VXM`@VYSz|NHlcZ|h-1nZ;fgCrh>po=mURVbE>o@oA}k$L zsatJ*1f#mr-7RY?|EO*FahSCGb zXx44AKn8}MECkMtk;FPaNk;)dAE`Ny4V6pfO{6Sa9xOn$yJrmMViDO=*%2QGI-s~2 z`*^M`i&}#Kw>^QfNt&7|12LkLgSnhVbaBi=p?a~HE7WX^0Sif6QWI)`QHl&YYNA{v zw5#B*|h1?)*&5x1kxz}Vzme@Xvq%Y(~_=L17B(&?C!w^uJ}B@eKELM*PLr8*MF+Q zj5B7?WSZ)Snqi8^l~VX^34NP&XSE+aonpH=}3yjyu*4n)yBpMd5d%qx&K zC>)%Ea*sGP%vEk(tW-|JdloT3zE@--9H7_*T(Vf&#Vv9s#}0u-h}MMQ^$poz9yEhR z?1EmxYU2BZ?>jz(hoB+n0KE()i7Wae)PxM-;c79bf+LVJQ?~U2l`xcZ-Zq@vEV|+x zQX`AZZc4x_DaT5!Sai? zSb_&pa6s@ZS8fP^4WHo;RBQFKctkAGG@>Iu7#0l@B+t!}18@TH$e13AP!1V&Be1Xl zP+V?XGEq#TSbgO09bd?f!1wG!Fph0jeVJxYYK82E8DlNg$Og{bTk8R@wWhrTbw|Ns zf(<>|j|3pLzgXb&IH$1aEO$|-Q_@EGf`T{{a)Vm4D4tS3yKfsdg{{v zOI&z{u09wRNU%z{YtPx0E_Q=gov0}tr6?2?VJOMAR#;B6t7_(KeFS}2v8GRkB7MAG zw}Y6l6jw6jK8KpPteEWfW@nGvTU1TX z?y2Z2)h)`m(;Fh|Wtb~*Kk0JWJO~axz!3`WydSZ^d@_7dE8<9%QXd>&-#`^u8cZo^ zZ8`7NgI6y1@$|PDyMi-?SIrf(A14pzaDMQrG5e0=+-()7QsE%t1ZWMXTH&?gTy9q(V3o4eRMA?&x4US6w!_sCC=d-x@0xsOq|drp#`(o8YL+o7Z?>egi4hs}uE6nCcQCGqL%0$FWO5>`P9zLEM4HCGa8z<=aA;{A)M?Y}LTO^XE@ zr85V#An8?=##Osfw~_WuW`K}QY?d6bED+`KnFFd6X(Y}iL75VW575b0p1nOIIz%W(h1G#>cpcLcxX#<2!g5qQy3ymH92x%z4oolN z@Oqe@`!UBwh!GVN)!vkpT6>+5UT1LF`YY&RUI@<~Msat+A}14_LR*h+PIgRfI556!9Z!jXHb^H|u_2!!FhjiWx<%9N}PMKL4`naZt%+7f~YaO)YDJo6dmYTxZ z{#QY3a$F7bnm(C1Ma8I=-p+lmgN*8vIL+YiO|V*rdh&;PpmYBjv`PDJO4AB{>F=o; zbeKGkzneM_il{`JF}9-jxt}x9Euk_dU8yVVy!Vn%2bu2@tq?F!kkW9+~yZrt`bl&GE;5f1s?m)WE>Uc{0 zB|s3>5uIhrL)w8JW4_hx#|%#)TEMYia^U@xA>SvwN3)0SNTrKJ>;7kfFGrJULb!fc}z~S%~u00yms$OK|K<=_!JQ1#K#iW6rpf1%h!f6lR ziPHkiKk#w{6AoKJ`@Nmpoy_Zg>~3~CEp4&#@Rve3b~CqL&4Z@}YxG++PIZ2(kFzuq zZNcblH(L~T_niJc(&9+gXhj~5^4WZ#Y3WBT3pI~G9`UDv>#a~zC(?7Sf<_s9G9sql zufkSQZ}RRa_=9$iwt#+=df*yJh4o46+y?JzZmo0ortxX_#`75VG&ZJz2?Ku$Q)i)~ znIVxGP37^2h3a&AlrylW=gVY)n&rKCC3SR^b2!6cCk27ZSDj&Xfr$hnU`-g5KC}x{ z$gYY!qQxP73M=jFz&cmKs^-BcBKL+uUMCct)Tcur)YqQY863R9_*^@VNn9ETn$|iD zWP1;{LXyYZGoA>5`@V`-c+a*V@Dbl?rS8_@h$SY};IbSEwq=+^rDilY-WZtW|0&qUZGbM<8VyTj^Lp2x`D z;q`r6bqfvJ70IWE&77*gJMZG)=h($|Xg>=cVm^WXdlo#E3t~tWDn>;bop$d@%yiC0 zdsReL=zK47z$(_FVdtvOEJW*woz5!7(UP(e9iuS2I;t?YCi(dgaLG3)co)a1%L~uDKI`I80o@8zVY#yK5)c z@58^RI`aPK+x>P8p_6+rC8Wt=p@}np-EsWa#bbBp{dO*lni|zOjuKvfMzv$R$)#zf zG3=fj@^f|O5^84V?|eUn#q@mDm<4`UjEkwA)2#Vkcjw1NVdZl*X51L9MzTtesM}6% z4!ho+IM`HNE@Cg_ChS=8{U&s#br$LXX389-sx#G%^s!So>ynA=*x}vm@w;(#q@16F z8KvLDzxIA+&}E=Z^AU=Wn;>dRqDX2dXg9P)Hy=h}G;QTpb%=9F>Wm@}r5oq2(QSuO z*FGl9JZ`-6+}!%APCeXwl*K1Jra2u(mK*NCzZ2chg9rX^Z!3zHC};bvw#s(cmxwZe z3tKga^2D!%!eo7pryP!qQp$(D=jEZ4Y5p^hS7rTn8r=<{G_;)%nnAjh3DY|f;`5Rr z%-gL=Xd+c%bMd~4d?%*q#D`IW`Pn+H(?0W^WOw_d)3h2$*ZhP3w+Xc;qJul3@c$gC z_K1Gh`8uuCnOL2y6hVvW=TO@r6PQ{+t;!*_3{;go$61i%63=wB zJjan|D<>p8Ew#4E`IcwF$ULE8XWLDN4re5(4rQIkoEB`=6%17A{c#TNh45A7&h&u2 zhE;&&nM&iD*8!bOYjo;%JEU&6M(%uMwEbA#0WUFqb}!tw?Kyu7Kq=#A<_eZjE@G># zkLTLqFcZ3Ix~|L}_+3+W%>&N0??U9qy1BJ(+$kh&J z+);1}-M;P*x5K^X5t3}L=MsB9(hV!oIrk%o+g{%#IKSKNJC9_+fo;#b{l4>n;PGwG zyZyc=J_Ecb6o&oV!Eabj&GBxJ$7|xi@UR$F7R{a>eWSO^aDyI&CWZDHb8I{7`bS|M zx$}wb%;6rC__@r!ZaZj&*j#Y3?Due|lS({XxV-~-l;0K|GrgVNIC${r=>FCRObiS6 zgI^&v6VbLMd%VQ9?&-GDyc>Uq9(w%DbnQX;5*Y5#!_Lw2i0q6tq%O>~$1%30g96C6 z)jNQ(t6p!_F5D5N0I#UIgEa_`a%%*mwguU|J!`l<&AarOdbI{SgqR+1^03dSo2SmemXz^&VA?rNfEDGbgNFJ% z#7q09&JdFfMcmOnnliBwI!f+ zT6~<(2=q}X6T?8?5*1Tu@H5*-_J;YkS2UC(0HEj+DuF`+;y=Y@zL zJ!Y)WZ|EEZ(dI6$z!SQ);Q0)cV-M)lM@Pq`_>_Y2u5;yh>P%wHVY+d;Lo?ya?eiJI zl|n(C5n11!j#QoH_7y<)0BO!6BBT{w*@!DkLAx>L>3{W;Lx*y2oqg~9KY9Bf|A#Nm z@j-H|Hxr8``tXt3#bUBA6OSeFiHiCDUFmPcQsa8bux>(IUL9ady7Brjue;KXKaPzV z>1HpJ#`d^wuW#GyJNEiRd;O8U{@7k`x*30l855ZKXISQ@ZTtX;PB%Ak*BnH%c1<<+ zqAWd9%^7=rMn^Q~dIn+%{0?HP$vzWLCX@XG@l+-?-aCMup*GQPdQh884D=>aG-V%OTRmRP!zAHqK$`Enp(OMNFi(dhmMkn3To|*1PCDXpEG`z!$NE zAO!VEpa+-4fQiS`#dHz#$1%UzJur|iCew}YrWb_#^RXe4`Y*DOpu*!{Bn4RgeDvl<4I`_ZUM=d8FFAwj1La5 z>%UjB$TiqQs`1tU@YcA70d~A>D;^uaFO%s-%j=QJ(~WQYZaVLOwDa;B%$si9(ANKK zTX_w&zN@YO&9?FyYJIOC$U@a%u%#WQC*62IeHFk>jQ4@)jbA2rr5Zm)?Yrs5t#sq( z2@^*pXeWhlCnv@eV3p>rR8yONFrF^<^pC~xT@mT#hpFZ-z(i91h~#a4h_<-DZ@kGe zxS%+J=AB&q4fboi$@^4$QD|w-yK8scVf}%yW#s?x8o1fjbEf2 zAA#*az9H`CHeuuTsQc2*Pg=mS16sies&7QVvDJ42=LT6c)%Z`K#V-=BAQ&Xhf!z^{ z6M%}9p&h`jQocc<_zuQ>VopSuFJ>13pN?df^yL1w(s&us0+%$LZ480sT2pH zPN1N`OOVHSdTR)xVJjd>vLkKAjT;9vT3reUF0vbCpNA8=?*b>!WrDkQ&^?C(WP`;a8lC ztEL((>TV}-=$cB?fhB6{TJI1RyC+~*H!l#E1|-e6pB)SCL>1Z8^^z%WaQeT*Eca1| z#(Ho^?4+7~RyVz$pzirg)ZLQ-reGYfGvDi8KWZ{O>UT$h=~Ux?#wNKBT&TBW`08>u z^JHwtkYb()dVdF(ABm@$DclTVt6Bo11+}|oV)4*q(ivl0;T|)|49M?6OtpYGX(sLf z++=JxfD07i=BbzCWHBXq6O@YYrq-|by7{K7K`b-R_1wKU_MiXXc!1U7QB>M(v zZy|0pmW}KE15!;D+w8^HOu#7ZRy7Q}pZpzSfdmDKe=)k`)L=fPq#aiPZT6R<$sUewJU%o`E6oS-LqVBfu)S?!X}7mQfZ2xSOQN zx9RuBl1_G{4WK6JA_$;26G=F15Sg0^s7Y59p!a4H=w&M@&qapmRH95jtDhV?A?_iU z-8s+$Yyh_wW~=YXa+ZIszGsb!3HsXtD1uqv1v13z6(!X8HQ3gg0FHqurJ4st2oJJ1 z5q$$HEe@lLn^fc1qTirHQ_aH<%|M^3av;1x-%!^{GbGwu`U7Vem2T54$+pSUjSt*7 z6b08GxFtRS1fUH0wDAE7Y;Dd74x6)HpQ3@P+^29i$p_wHC3UNTNB;7-IwmRJ`_GiolhREJt)|)rTA4dR(^*z(gH?RPi>D`FWUW2|MkmAsEk+M?iuEhr%H zV`UBK7GH#%xLUY-9>{CLz1JiR_=)fF9+)TH_z6E>+bxVVA^dnP_1c8^dFrcz=Hi{$wkQ}!wW4%VB|FFEG5{UB{(d}u&>Vp&m% zs^I{^UQ8`2QHy&9ATRJ>Q_W}O3RSR2&?NTwTDtj%pcWbh!&=9E&{$uilP@D#vzNVu z8V7Yogy^h=hr$+6BOY?IQ9~iLs#koCX1u4TcL*OeJF4v^zV7KYv3QS)`k46}1B2-; zXmAXE0r){~0aNi8i+va7LF#eK|1@(Ej#a;q1Kh}wAvmBItMgF9+9F{cjNFBmbDfm7 z-vvgwS1b?dO>fb-ydc=%&Y44@Z?sl0`j=$p{;y+^lIcYv&h*Cc5i>pg6MgZ1{7>}sBN^47F2?)gs6;|$ zVw@i5>-Uf|TjTa^)>gIOA7FWEEkLD~(|AQZY&RfHe z@ey?Q{sCcVb0*p=p>uY+an|V5ulzNjumT8a7NHA#sar{jv*4(R0pr?F(nP}T@97DJ zV9mK;$a_FqdW(92o*YMiOd|Sc2C1EyH%#=S53)j>HjY8en8snbSRJ2` z3!FhVjKd&2a3q!-WJa*@I{u@ACDFFLfp&eJOO7T-lXuRJKmp#l11v4v3oM7+fm{Xa z635cmGM?VamB!4dYZ;kgEKByIPj#k~d`-v!6V+0Gqhmej?dw~EQnYZ+RO2_u zq$ZP0d|ii$lgR9Q0~rYpfm4YY6;o)wu9&(mnBomEbzNKEPB-4xR$fs5w${IsZoH%Q zyrBLat^Z-V@k6cW1@%8nAo^u=d>zId2n%NZQM&OX?a2%3e*|MKv>6Th{5akCvG(Bw zeSS>(ak3rg8|;gIyW039f&)an;9MpxZVn>=c4dX{CTexYo3dNpuqUSgUvOo5xf1wd z4YXMAL5F_;354eWM@Td2I@Jd>j=l!-eZXl!1V-TWFpN!Oz+FHs3>xDYa_24Wack!& zVudZF!3Eq&lQ8jdkyxad5TAqJ(nan1G4juli!ny3@4^}QDS%Y^{#@#A*gC7o#D%&+ zzd{DU-*<)EuIZDVY`R8)(mrEf6OXPV?S; zMteWQ1Pg_Flu;#3E|dfUof&`uj^lLH34l_9B9?UBIiWh*N0Ue(IipS`*PDQP!r8_I ze&fA+al@Jn?vl7m7x95)3_iy30e@;Q!x+g+_9PGqqYD^`m9p7B`dkUr5+;5*Hh(6m zKNsU)c&VD*z{`_^mm|wteQPoF6P0RC#?0=iGshS4{z3VApfB0CUO#>Dz!coo#pFG~ zmq^&xt@D+4DGtfFF&T6ni98P4pl^7 z(r(OGV`hYJH^%EY@p3!7c=JnZ_4-Eb%hS`VcxmqD@>H&}KAm&8!>qL`H*%Rme=U%n-PcjVs(9~U@wb! z+lI14!o60U(*Re!TnWuZEG4uB>7}PArNAN=X%B9nI z>t+$uMaAHJ8LyqjJ4*BH_S`+&>c+Y6HruY4`4rz!O#Y9qLD|^G;2qd=``}N1ztiAY z>l!<@0U)GL92)~jpCPi^AblO}+kliCY8i|v|4Vh;h?yB80wH z0>;p-v9}YHZ79ju+3vyt1n_QqCT!avB9#rlwo~MTEYq(qk?ptN2H~WMO-`_UF%INg z&R6*Xy@t^o{QUPg)v8w#k5WiWYx%IEz>AwN z;{ChR8=K2`WA}6pZ%r@Mrt$q({ECg*qq!PiKR+G$0*PsS|HZV`(9@ysh=V>4&m2`~ zQyckZvv6wmz`;Yt0{yxJnuy8XeCyO}Kg<5pKfPKXnE$dzN{;1BXxE|QQmCIX=a1p5 zTuY_Xcu~1o$Cn%7>*3_9F6@9$AI3DTC=5;Z+u~mj24X^#`;2Ks*H>i+W7}5Gey8yJ z^=D${k7#J@&mZ-|O)xJaOK~2b$8i>E0mpmK;_pw+<9Y@G`&sGvlivUOzbI{p|K#3X z@i(6G!zn4F;h#Cu|XaXw#Yzyp=@NGz-%3BaF+#NsWYm2V=fp7MI9bL(>N;16Ib2Zb#=o3AWzDfui)F*9WHnts|*<3-yadz$FKt5+;_pA z|GEH3o*#X;kR0HjT;L}!u$;B+mP{FOY;K5riS-19qI z)j3tS?(MrY$4$FJIZ7!HKKu78^$@o5S5A6pQU`TL`$sa=*3_3eJfz3I)M3(@bE}IM zR>jV!nt687teFcI#1<8uHm9iS+yzB*7ZeRUeqzztvDtIV($fQ-+|c8PD>YU-$~k5E z=H}L#RL7z;-BT%}mGWC@?+)OLz?WmIR2F$-`z-?LKmY2Whg|*~Rk=n?+$AgH#{j1a}2=>1AT1{$}{hV!4yz8{~1CLDdD_b<@{6)b1HTnx3b9MXO zYzA>)^ilZ$JlH1vm817%RdeRYU?_c6=rd~&_=(-nBsa1lsVn}ZZ$^35 z*ea!F->pV{x*4^5IH0RRVHsPM(Hs8F>Y;K;+L7K<(PHvtM&2zgno_h#Z7!KQM&uz6~ z@k*42YLAt|S)i1LR?U5$7Y`PZV7Zs(qXMIu~GyOD1m6z9H@C)_6qjBw$TnU zW0Z>YG^!l43=<_(-4U{YuL}IaSTXo6zN)-9Z+x*%kabJ-wJ(ZwrnY^p%fz|>mjrT4 z%8U~|9DMFRT{6x%W$C7y-#D?TsB=s5N=8Jxf;BO;9Eg>Qw6Af_l@3{(^Wc-`bPHPx zhj!6!&E80%yYS81qcK0wPnllmI#X`miWrSBKlsb88FJ1o0Mjkgxm~sa1J|Dn(;BjdB}#bhatB7nP@o_4r!krrbgO# zNc*)p|k)B%RAcliln&=SZ8wjg?xkZ>g zb^<^rIbbJ}V?)Ty3;5a}Gs{;!0H)2eV+44a9bGb&6CEXr+oDGxzAa1aNU%j+Cr`e8 znk$t;QaZKd6n6mAjU7Sjo?G%vQUF`m7*Y*%%S#Gj$AWT{NG1-ZtV|rtWKNvst%#Yc zQ8I6^#$n#HL3w1}$X4dfIO(^YHz)sO-iVPC#Z+?Vjm6`M(xJ=&2BOKfHx#jsyvt8tJS&*>vIiD9NS zPai9lu2RbBBVFkx(?^&n-DUbD0@$L)2HNS92w{hWa(DV*9h2#Um5tM?Z�>jHh(Kj;C-dgydBZlJUe!@UzBqCOUuc@st@IAI~$TEX~V-D6o4x}p_Php~TPb>$CLbO~C66q%|2N*0FPaKmtp3PekCyysiU$7n&O1l_O*~)mH zEgiSx`RGr^Q;eMG0+`tGv^YjAmium^5Y78;Uy~R0N-EehOXV2us9vapALE*-q8Q%N0FQg!e7-1zrC}At zAwLi;2dVl(2KR3uF&zEmkO+UH4#PGT{w4}q8gS@e7KEpSV-$B?_9`4wRX}WG6<`9s z{F8aIo|YRd%}?>?pKSd3F`NSg+GjGplVKd7F&m{#X**5g(%&e|O6h~appSE|bX4{q*~}jDbzbk}ph@cDW^|Bw5s`;aHvU;{-T!!~-W3+e@I z7jS}M_cAp%?^bUW;fZdiA(en{*@y>7-?dUZPPyn8RyXuZl10DVfKi-=1WJ0#iRv9t zXu7zA)K5d?@zbmR+T{fgy zG{Ut~d?3A$s~KqgWU-TL7*dTG zHE@qj7x6$`%)r%;=nMg3UEL&}5*`OK>T6F0<6hM$dj($Zg&ADD5e@e|GN>Y!kkaT( zks{B7<=d~dd{2YZA@aR8ItvC))tIFf4-h4i{)spIK`uS-t1G*36bbJ<+aWpm3+Zi(ZG{#^z^&imS+jyaewxq;FQ z<6H0e){BKNZ)(72E}Pr?%RCfX>Rbdn93w*QLjgxl)G|~2>(hPh+fPfK8t~`g{4535 zU-(LVyspp>dk4Nq^p~CFGP4|qO)LlV(eYYpmg8s%!AnN@TpMd*YD6vHBm&mN)JTZv zpJ2#-*!>|7{g7h_4!l?|gvxs1;oL_zRa^IBW7QN4=53wx+_0x(x4f-$e$}Ov7Pzsk zv`y86B9(6_vL89TP9$})zE~;PwO9=LQk`pkv!h;w><*puvRN1K&17F4g|$QFCt^fw z497-VD3MYv$lj-0M1-Y8bZlHd8YvB0V4Dp1PDK+OmojV%&<0MVFtF!VBKzBg1bpjc zBU02z*nPQ&%kIg&@*cBJ044>u-NP1J6g@|ZAV>V~WtKd1fNjoCY-Mj~HW$9^l_-MS zW%0HAOQogGxm6FNz^3|N2)T)pys90LEE|AC6VmGYWbIToxBDF>^D|^3xuM*1F^Nnh zFO=6UBb0}0qxZtFzsEqGE7wj zS+5DAY7=S$RhzIkwo6oFtqLbSCek+47OJ*kU+fD}4Y#U^qACm(LRA<}iTzhp{jF+} zs16Gq2GwCEfE2z7v}LpT^i=Y0LYvnk_PU?Lqu9ii$NPLE9z)#p})72QON zL&Z=PhcjZcMD@N^VG%cxPN7avbqZ(3szmjQRbk4QNas*zs5*zUVk<X{%m`ImU z7pS^~L$NDFwau!qyqn13p~Im%JRFYQDyq#^h1J|dqM;~M(QqVokEk|S6=tf5bPaWd zs%tnq_L!(Hx2l<7 zx?L{Q&7J)*SVg*rOGDkGI1?=mb<3L?=@#miipA^AaA|ZA)cDqxn-=nP8yYTaj?<&G zbI23w@smJ3!)2kKHc$@)>JjP@!V=s2^>7&@rKOaFyzV054|%yjVC4?`_T{!imn;sa zv_SqAD76I&v_NSsP_P9`Z-Fvepv)F1s|5cLff9Y{q(j@Ovuf zqU?t_Hp|I^nDAUgmfQmlzp_B`h$!bEvh>6#`N=uRc@l%?AhZoZTljtqfaK3rn494=1iLi8Qqmh(HyR=OH0aCpZWNC`jO=I^`Dn(`Fs4mD<%Q3YAlEV<&!*eRR;DJi9h0Esl z@H@WPYDh||dO2#LHEF$#8=YlqVBEQTUYBLK2ol)m`weLIg(9mHO$;a3r6tP}5#+K* zAc|Az(r6Ayog{{i%DD~J2l+maoN>7$&^>3_rPyGcF?KlFYa6>vOy$~=q>St&m+tUs zFFv-XID*G6hnYp)5+OWpj4LDte55@b)98C(JL_Q+#o4>Bq%4X#S6V0iaQi6NQHm0M zY_sf2=nm*(^;Kdo9mQ!RIx77<(rwGhG|{V}#;(JTy0BZ@cpylwfx)5rdqA5%>96QZ z+SxLebaXsyN$O?K+d82Ca^5D3pRf8$NfgF1&uGliKe%lGyMowW+iZKCFbB8A4Hnv7 z-)wtB>$W3Y+Z&s0Z))9kw%Bs~Fo{%0j)nN)d|fua>S=Pev_J}dGPsz85`d$p1+DaV zWjrp%R~5T8>Vq0#21apd%$L^8q*UYEfIV-QIoqkFE!vs~b+k8J-~ouQ3Kq6D2K8?K z8zgNRu$zTBcwN7D$Lp46+gn?=EpX%3H`{J(-4@^2I9|6k+cvarJ2_sfR>lC!wmSv` z!Ntb_3)UgWpgJD+XNr-D0-iOX$1Rnq#kyem2mjx?8Kz0*!_Y=I5?4GF|boSAQJRP_6wPJE!Qzwn`{=o zl4;QgVY|FszPv@JAhXz3@jbx&eg_b=6$W)!RYp#G3xS;olYhrn|4|Sy-2ta|dH)V4|e>qGw2I=3MNaIxVGJN=o!h z3Ys6Dko`&?(L!z9`$PFts~Q!{vUS;n8XA1tD5IjS{`w0OC z6E1ra0_?^Re3lSgMZw<^0-Pkl;$Lw==^|g%_X%`59zOCk_n>sDuPP;hMyIwf>YbU| zep%0eFV$9~v~Ozk2~?qdMxRc;s`R+6FE!!`d#WN3L>r;E8I}5*{p>{>-hn&j!F~Jp z$II($9tzBNM}33NmSQZLuO}A78!)jarP-{WN9CrKmibId>?z==Tu-rWrC5u+lX?Vw zRqf+`DS1=VTvhMqN|#_N7yFF=K<8u+cC}sacYIZy<56tvb|@lEUx0D8 z`?w?5&fMd5vyI2z&Xc1xj3Pa~QfIc6*L=yO7t&3fFv*ER(v=eVnqtpKa^TkuV-!}1 zeFWP-0?O)(4`{!ojmO?zJMs|iPj1n^oGcvJemf_Mv-?s!$%Lf0#M?VaF0mG|j2nlxVdmWIyx8pYM!c8kQZ+?eqCC zODqhX^QvTGxSe0rLT4`(+n7d$(f1zcyDXPDU*dPCZ!u+kc4_(!Eb8XSxvuI7yXp_H z0tI|d?P}D;S3|nb_n|CH7a|=w@s&p%oaimmZy)FPc$k`Knh45X8HYyV%ZX?>zsanu zx)q%^(FRzq!w#c)P^{VvVe~}=M;W-@$QpsQTjs3;(*xItyJ2q^0?b>9#d$07ezHc~ z3GD&%_AZE;=dA|st`RSxeN!XdjQt6$T_cza9ey%zo*~}e#a-*Pc*C_nFUpkv3?K0^T5(Q)k?9RO)i!=Cf~CA z1MmUmc@(09l;;(eM|u%^HC`Tel@6_yC*kJQ(D}8b@;oL@C6z}EYK8Dw1L^AomgfYB z;?oP|OH41(usmcIRQ;Ldc@myKw>+OEmq#3v%kvEI0p)ofqE^czy+|q#yGn;cmq*;3 z8alt0RGt^5sig9VL9Gy$NBU~ZgY)-9c~HJYc|^nVkXcZreX{qx0?(gYo=kYecQtWL zF3+352bAX>hz?So*ExHn7qK_uvxi-!Lu=(pxH&a+elw{&?@3chZzBYX>O(m62qSvwza;sc=zCwD< z9dL4T2~pxi2}Q#al37_biY5FC-mR1{zP=x#hLe+PD6v|v;Xk20poZT;)M^c-n@Kff zE_66_4aLo=p>uK#|07K$)ldv-h48#YdY-7^i4e!9A?lo{p=ek`GApajU=4qS_b;en zEV+gftMwWh9NQ15p%0=~Ybf1Jsv&cs!=Y;^ZcYuIlWXXgrjlwX2DL(1L+N>Pr4MoEml37`G6>HcQ-oK!RHz(In zVzpkwB4`hohn*m5wT9Boq#808Ivl!&;^x%Q`R%0ju!}U6R6{YS6~Y=y&l5E~3F3GS zQRhSrMZ+4BSy^>AYghvBUr@t8B-c=4wO+#>&>m33J`lB9L+NHx4Veoa4qZcWb86_E zT*D)!siYc;L9GzhP3YKS@~YA71kkj%=eXIaAm@csogd^Nd-607wZ9u4gQ zH5>|2t2LBvCe@I+(BaTE6gQ`a&dD_#CQT*PPz-8?u!hp}L=8`dI6e1I+5nF}2bT|;qmYUrF?!->*VQVqqR zRtRe-Jx|ndI>hlBqRxpLiiR~Lv$E=Y)^G~Ee?bksDM=^P607wZPKWk@8dgHoY7M2E zNi}3HbU1Vk#m%Xqb8-z&lcthtCXhwL(}!>3O1t zr$8K^hp2O+hN58&$*ipE!5W?m?_W^ELCH0gSgqG^DYOUFa2Z6c)=;{cR72)MheOv; z+?*OZC)cn}no6pn7}N@34W;La8lDPqyoRWAqK2Yj4auymI*v754ewu2!;_M0D6v|v z;YH9MP(wUz+joxo>v1I+5 znF}2bT|;qmYUrF?!xyBfq#BArtq|5wdTwhd*ZabTPf)neeWs-h&sO~52fpOm3;eP8 zV0~8A_*{g~t@!*NAKd&=yYcxBA3PANit*`-&nSGR;}gSYB|g{S(;xQ&)biQV+vria zGoer;k1>+5Bj(~mpVmmeRl`j4^N1A>4;vs6}i%R@u zj-Y+_Vku7}z_wP%-8kfR zg@sngy*T7^g@;)ocj8crD{N%e5tqf-K&rba}=@GzPP%L{NBOKz4=AP z+TAcU(T_lD_kf^nS$pcx?<3@1RE_BQylGz97`)h)V;xM*UdUth;`T9!$*uhao6=(p zpIl7~O6-)@+8p_A?OhTZEU6;z=kI z{TxiZ+xY3X z1+GEMZgy@{6%qqk7hHq0TK1uBReLdzHR9j~g;ia}K-P`~~qj?})OVWqo?~=`}zkUWGwZ>o6|G%ar;zw!5$x6;E7LHFv=o z$WW;_E>^1E58332YSC5pcRGuYojht7bgO}z3ZNT0KZdU#{KGDyr`&p5T53DM|LQ*S zaHq76vk!%7uCSmgtfmUzr4`Nx6y`gh|Lo!6dn0@&rv$RG_+V1A&q$FmBu+6tZScWz zDbEn%?~;hk&Zn-<+?(f94`m7Uq#vG_sXY03wn^QTUK#SKTeF_d_Nl8fgnphW^a1kO zq`gRArMx0tw6lUj?+Z$_7qi}u_*6}S&@Y2RUkF~A?^BxmVrt8R^B^xt7kWV(v3W%a zT_2DLucrzBOwOo6pSpxSj0Qxql{UL_@?f){{CA{x=bUAHs$H(o&6Mn5%#*UkYbrCn zKnwr3h)|#I>-DKKDTx}97}7Mh@>E3PJf*~DTDtV74eiHJyC(M=YZ6!{m-@!pU`K8;=}v>A~jl6~Q);nh1=_&xbTI~EH4kbQfTy?Qn*k{@U@p+Nd@ zBIWC8`A7Efj2x+X7GoY6xFE-;ri7$o7gBO#;YMixoGtz7MoCHAw>b%LPXvFQ=XklU zHuymPkT}XmzJg?CW~q^={Sfe&Zb0Rz;}HB_p_iySvobUU8XwV=4AJ@ISCnA+7orsT zy&j;;F)N1Py~iw~!x%GN-hY73FlL6zQ*}UJx7nLF6TgOoi3rr$)mij1V`j)Z>d-B5 z5oR}#moYQsD<;s^^ed53hP-_U^gL0PDkAz1QAqX-vCI_9h`fITG&9ShDA8J4cEIz{ z8oKk-hngX(1D=#zfjmA}L(NQ}GL;HlrWtC^1{y&0Z!M^k8cmc<-Qj8)(T~)XsJYB` zZA7AUSLai=zm53yP!|y0>P<#F3kCI57gJZ2BdC|Uo@iONpg!0)G%vRy zXAiVD6KFRrgSmYoXXEz>zE&Gyd1=l`803COPeqRHf4P7qLbz0CM>@T9bh-k2CM|5{YbVsRrkbcc&_zy9RQkxx}Oa+w*{Kt0>xUOsw6aAou|E6?!Uoazgdk?OSM7FsX*)0C>Q+@ z5HuQNm1u8_x;_bwQyb_vEQDj58m~4IeSs+TYJ$3t=wj@K_38xm1krlzc=dQo=1)ZT z(r=P_pXhG-O;%qLbwqyk>O}QDQD>qlIGvV$;&C*<0etF+3F?ZbjVV@y$W9EtJjQ!9In7RJ5$vt{45L5P@>i910ob2x;3iFbWvyL zFHGyAP6#amJuiEIY6taW%3?eo{I!2^S_gGJ=>q>!@VVrBgKG7L|54|}x`hRc(@w1G z2HHUlPF)82A$_N$ilms5OJLc|S5ZDVbtSz1OW)($NQ5uQcOlU4cyR z^k16RSB*1QgZDz)SDkBafcAxy8Q#9?z0_OMeCi*8#h~}3H9)(+O+#8Ql^c4%*Gr{4 z15=u5SMue7w;>T;eD8#1Lefi(%Zd5>;BA?^)B30i&)sQ0-yqK-^lED40ni@;51`!% zDbjjc%GIzL8Tte0)X0;dKLnluea*Z8`i^-EHXT!>GOJSF1AhbQgDDN*`={&%olm;k zC;YdhCy~ZTzwnA&lkZMiLHZ!+^Q3D?H;}$b`#kV{)Mw^n(Ea9f&>Tl-ALlL5znSkq z|4sTw(w9i0P7`l3qZ18E9X1q}lDuRSVRUsky31z2VcU zBDg;lqgFTrOVPz%;h)Ialj2j4kY1SoHuzPb8W(v*`v9m+0T5JKe$AWe0`7wf>E;^r?VF$S4bEOa-p)QRq@s z2N&f;9yckftBX2CMx>;u-bsFbH9C%7z`4~)uH`iaLxKS{Gp<8xYHka3P7Ac81zOPp zT@*)Ih;qF}{t4c`PMWHBQJ{^WJ6)9DCJSeP_b2%U)pK$50?r5Db}ft0&kWTR*Np($ zA4g+@St_Tat=WIG9>+Tvhq*}n@OGL-oD*TyJ&v;Q_QuFKni!0zQ(V89g?*hIb-IhL zDVP?_Q48XhK76Y>uLW90v@`XLHm5teY8}xt>Y7krptr3~4=yMHdXMN@|F*(`PM-RU zi?#q2sPA1rf8cyiTa_+f$J8@wSN;fZTh-Pgy(@pTg99f2p20qHYVa^MfV$PlZ;ZE{ z8WKl7{My4PBH16#47OJ%5v}qcle;FRgPQ50?{fsrC;CLs$XO8Vpf<%(b+DuQBhj_~ z?Cd4MVwK-Xn!ncX1?sHMv`8HhS`j>4Eh55MZ`7!V2Sc}wk>P{>ZUS@o>5YlGS%NjQkOC{p6C3BT}dR_ zZVdKTH@fKi96_60KgqVY+7(B41pDv>aLF$_`>x;-sduswp)Y!)hk3c z+W{&YUkqf#WZVa+BZ*ch$!~z_PbA}hS8$*j=Atb?gVi|KPx3ojEq0OQceIk<{!vQu zdn9;_dXmVFX1RKm$d2Yv^^S{VG>59auAk&LR2AY&ibR=_(-^E!#}G+=&jg1l`SE`{ zn$HABsCpus(MYwI$YwNBdAo|G&1j_Z6G=wT1V^cC7i|F=quLSK5j##D>mteMICUzK zWb~)tIJKV0j@a>PBat1k32L*8WW*+@t*)PBG(mkCM{fjAP?>UBgCq8CaH1OPB5(G) z!O3bJkC4o`MAGX*?{amO zMgE@Q?>%+uCKnwaF7(!^O+?rF_qJJ`zCvw}qf66Qs!yz?dN%di^i}E`BAd}FmEA*f zxK@1=+5lZgqO~d#zCC@l>f!1dL%#zW?CRbNZB1XJM!9Hj>LWm>SWEw&>>cTA)gl*d z1G-SHBa)iEoPM#o$JJ$Lzm+!iS zoAn9{D*kM(*Q+5!QtP+UuT;mmXbaHQY6_99^|flQi=@`qsue_1>#x$UQ#TN;Radt8 zSNip;foOexFmJy126eyHaU5o7()suzf^RWtIwPrpfh;GzjY z8`R%jRG9yL`pwGeWh-A<@O}C%D#t~41Kp|)w+N$YGV0ZUI7-XdsK&TxPj+_3ZEB{A zwgGKYi;31^q_?Kup;i;wk-kH{NA!u7>fWKga*fWh}h-`K5RFjFM zxQy3{$-Oe}Q}4vlz>NFV?l`K*ctCv>M`JS{R6kgxw&hLE*sAhy=#+VVbKdlf->Xh8 zS`eOZijYP0o2CCLRJcW9mbqPjL2BlkvFv)ap=|%QBu&-?^4rU6awM_7ln3({D4LRDRs> z!_{}72G3I}kI0_a?NG%;YyDyG^Ja%Cby0!$o{Syp2#c7}PBo0kX0%gXL9~-+fIHPT zA{pBy-lx@5F2b3P_gVFp>n9mKr}n!@GI~y>;}#!|K*-?|@AJaqPtE|hWxSv&sFPkd zX1u7T5Xrc|knxh5LnL$JwTwTh)q?bT^_2JBj91iMuFfC$DC0HtHj$knuQR^1@Sn6# zGhSEkQYWk8e>2`tpSylCuisJ`eZ}|_E%W-%>Ifn`uisYViEJg_RvTO-`Ms^0T)$oU zCi87Ite-@&=ezHyWklEdw-lykzN@ylXdBQjwTsBk-4E2KL|3W5Wk)hUP(}T16lvi@ z)ilT=IV1Z}bsH?m&a4mBkwh{Mg_%uih>NxWeWWH6*}m*mvxsCab;Zd%x691Gs8vKV4i990s@4%n&3x)Jbv2Q#?q|{w{5|893MPX@U&_!BLb->TDzY_Gpl7rHv}`%Z0lk@)>b{X5C;d)4k3iDLbJ zP~C{6y*Dy{P_tb$Be*;BM|G!*q`m#>Pf31CzZ>`aTc*+<#r?j`)cVULKS!sOOMBA7 zzD$qKbkU5UH_NN@T_i1}=sqrb*qff^*Ms7g*;%Q2SdwL$J~eJxm=)A##4U@n()Ge5 z%S?Sq+_E$)OJ5bY?41?T8@$jK*Z;>%lIX z5uBJ+pclGGGHR@@H^wdR&+4VWNwVytGb@@~e>zP`vPj%6Z z;HK0e`fL|TUn=y{B)?&LecbQktYP}rxZhv1hU=|Ke#h!JlKe*Mx8i=^W{uQ;jr;wW zHA?%3+qy`5V{|7MJ?!;{#_H2uG$R-e9j6y1Sst%f5ZR2z>+9owTfO7;mbl+`?*x5+ zlHWxAP?Fyy{Wj6HYH6q_G)d==kiJ}{M&co`$$HbV7F`@F37x1njkNuI*xM^KML*)A z8NmUesd~GMB!_AGk8#VPp_BEOam&%6={jqajUtw(=#egZ*gG+Fs-Ewn8NunHN_|C= zfT}MVs3$`w2=o}&$=_R2v^kFWV5nLHMQ_Fw%(Y$`n z)2Al+ouy|I$q`{)=q$aE$R5Y$>kF*Tf1~I2^!a+Vi|+JX6`HTFAhJi%v-K|P$1|j} zb(b;Ho*c(Egcj)DF4_XLP>&$8=ZaN&j*H}cqDog2$x-dHjB0&>ML4S65n7~gcP-@% z>0Es`k({mF9Xd}xMI>hzTSJTWD;7bwJ#@bQkjQ4VM1Ns*{^_0_p(XmCE;`-w9MBI$ zHlw9_DS(i)IAB53SYzc9G1p3-!ox z64TC!i}Y9`@$-f+((_z2BbX7sST7=4tA3Z3A6}<#C)%y&Av_3;!Rq`AJ?DEa*RQ*1iKj#Oa=nYl*7FMYzj(Fvyh4vYUivBZjE2|i6J4|g z=qf#r$ky{3eUXc#p4aH>h@_r#Gq2V47NMTK!`JEOTuV9cxn93YB=sB+zCnLNB=sy0 z->AQ~2)YsBoAiDno6!cHHePzR*8jX`TzG?yxaf7yWS~MKo6*g>KatJoW<8fkGCDPU zi)$5B~KRE-uP2cGv$>=t{jmT!ypr5cv&g8StXf8QrVjB(fRZr$4X=8C@K{U#FZPy_Pw@F!KSONhBGq z4?n0o5y==W4sX?E7D4ww=I`}UL^h*sdX&}qfACx%-liwGD8+jV&@>{O(I51A){hze zLI0LWGWu=!A^o6>wg5e%pLP8tqwV?=7fD9j^*2N|qsR1*79pei!;kAO6B8Ldq05LQ zqldzcx`IeDdLsO!KF%WOo((^xrxV$XcIY#$&cD+0Qh0}6=%UL!uK_I~vKj5v8>}BQ z+NnPwl8oLBKdt}aB3vH~KdXI{Y|kX4=X7TmNk-4Tv*+lg#Ouj(f2 z$BbUpC6lGslF@&{uj#%n+5+^39!+FdhPU)w7fD8M>BU4gqd)69i;$5Md0TIGEhVFO z^!-GVQ6TcJeul_q^qzjhB4)Hp?;)}oy|4dfb^f-<=zaa2i;9sE-my5*)>^*ReV{uL z*_Gh~y?{tE3PnEDOI@@DXt%!1^^-B$qwjW+WVA;=N@O$It9M$2jM_#%)_-*^C8JOD zzlbEGj*-9Uz!d3=&FE8|ZxJ*4Odn2UGx}Wju{wV#GWuK(a#0^-^tm2JWXI?WeWvx} z7=581Ad-x_M!wXKyJ!p0SNb*APcr&i|J_BB(bsxEkFE|Rqi^+DL^h-E^kS>?|JHda{GDFzqWhfJfG#Am8U0(| zW&N1ZzxB67lF^XJf3%TD6}fs3iF~hfiR`TVQ6K3d$>>L2L1Z)9r^i}^jK)Ov>vLU8 z86#!nR}pZn61_elqRo{=HX~!|En-HFxtGXhcL^h+4*>83JrIp!AEPi9@7dCWzUQJ#5$$Yzvp-mnN6)kF%+kFKR;)W+b( z3H~Ibm65il4Ux^LkUw6IelnxOOh4*uM(s?6we+8gjM|woE;4DKdX@(H5YN<|EfnGU{aX4C&c3IPV$Z?PM~EY(||;zD3CBsz?_z#I=-+ z4mYESB%>Q6QF98B&8VxHZxJ&pG4d`u&UbnBwbcB^>inyaQK`AyMeC4JskxrWX4K7W zw|>m1oAI3@Q6!^{k?tn!qAfr@Ocx?M>w1~tE|QFTnTbR;quyq^MabxPkv?XPYbhBW zVJ;<-j2?&_X>KF38TB>yS;UO`nJ0*BM*YnTR_DJ48TB`>yXY2V)ZgqPvKb99>Qq*O z84WO_i6oSh?JY>TuaGlhL`K6+5s}Skgc(L;_oET! zG9t<7)yT2tMi*@X8fETr{bbgSF)z7DG8$vvBeEHdHG3^WM(;(AGwCxG{_N^K&g2kD zMju6vHzhfxM&N|iRK@!pJX)Egif>ll#HgD_Cz+LlT6eiWc1I-G&8}qlrcKloJ1rU z{Wmh*EFiKO%`g{O#Eeccmk`;EPBqtAo&N=l(W&NE7rluwI@R1sWHYKXFIqolRB3W& zNk1hcojucZbkP=|S*9qZqxt4bBAe0KCTq6jU}xRgW*U)X zRFJ*EobIA6Knu-MBAZc_xxqz}QI+{EkqNHQ9ly~+$E zvKg&5BQ4?>tua%GY({I%EUWXMgN)Xivs|TTgUDudsp)SKGMbounVIQY${1a4<`PLp)3UEH%ZO}7 z>&-gX?_uxE>?_S(bEQ4pFAmPjzRD~-Tadk4agEtXv|G2u)rxD)79xAK;yQCz3-pln zLwi1TooOUmg*?I7ythckxxhco5wLEt6wOT;_F;e*t0koROcDIqffXZV}WN&MrHEKC+lKKBx zPi!BqXXIS023jTawi+qygVjF3dj7mjCE8E-=);oMlb;)>c^(EMQau|Xd9P3A?MmVc zO6%WYO~aGT%qzJ*c1$hZr`FBx3j?|C9ZCBCRi<}gFKa%6|4;WR+4pUndB0LL z8|QzwCU(x${z`qYasK=(MYD1K`71@Uank+=EB1lwp^Tl4bLFqpHydZ?uN2M3Is8|O zX5;kxm7>`=Tb*BL53+Hd`IVyCIMaTmXg1FLUn!c6bH%R|&BnRySBhri;J+#RRaQkC zC+Am+X5+l`D`jfq{PkCgX5)`=%YLP3Hco3t6MNLi;#w-_rSgWL`vt4TMk{NR z<;|D32% z`vmXif3A9}ukhymKM?g>)kpmozb3Q~@7EjM7r!adPX%;;m7xcyu$DLC+<{lhiG0vbqf(ci^hW7Q82Yk2*;`fOn#|sgunov z{K(!@$m|(hkbhoPsz2dv#zKQLtc!N-`+=M`-*0{Het32|4 zVlUDGr1*UYNJf!PAU%n6Ch1(#b3k)YYl*WSw5_^<{ASXJK|A2dSjU!S z&0Typbv+~xnT@&~t{QFT)uOTHKe$S?SwDuWL!0$~a8+oM`3hHq?$=-8D$v8Q_u>5o zp&QkrK%TRk{BCt|YA5FcBkuqS-_v=LZ*W~;{uQvX`S|R$kfM%J=l1IRlzZ$$e4Zjy{T>>gK|Au||hJE;k zefR;nZ}t9QkePGBX5Y$G}q(JTgmzl zzPZQ(zlDLCKjrHH`)7THj=aTO=*WA`g?Ov+b6+RNm$%E4i2Y*3 zIaXglz7G5|8gFt!ei8V$vg*li0{@lXO#W{0C+PD%Vxv`oepP=@zR+8s*9IP-%{JIf z2|P;viIlNuU8}K(poW6hsZnWuwOWk_y+Ta}-9R31B&7xLn=#WsA5ydM zm84M52QAVI(mLb#8pfNyCF+fWGQ7{dBfAX0k@F>lEEKvHAwIR1MB4!!d9?=pZ!% zv_hQ$Iz~MQI#Im^dNTf-_%bz1wFNy(4b1Js$QAhM*QL1?IHE7bjXL3HxqOX!BDWeg z%X1fj>fj1$SGiQ|SK-$*KFQrkNrOv8-T--e-jkH}2R1smhmt+88J)L>Hs9cUA|Xdu#OQ{ z>8sS6d8??siuM~RZ-Cbac@30rV(Smmw~^W>Nq4~Jo4g&gd7ZvZl<%hHZt{Cz(>8w( zTiHX)&vYBLHTR#i(MIaym>Sh9-!W2+R7#3SJCGI|sarAL-4BDxC=}DOlVMFDM@?us zgtUV86|}E_{jvEKw4X>xC2eMs&Z6Zk%4gAX7A;p8>ER06tb%+&{wm5>v&GfaHqf_$ zz74RsCcl9;o2YH1?+(bf=kK8WbyzOX-2=&<{JpT*T<}l&Iu2)rBiVTzkw?fEJ2#;k zoha!{NgvSVxfPTTqkK5!6G>-LJ_qvr;9N(lx{8w3v|K}*izu%r-2{0`@OH`@(TC-^ zjg;@8*G_7mX0+#Eql3>;+eFJI@_T6c8MU8L;&`ObUQZiU9PH{5udW{HNjd3oNW#Hc zo(2^tnB|eIXHhbTbS~}Z(sC8Gt0-SXT2FZcF}9bO&vorhE_iJuW{_ zFU|d!{AaXuybY>Df#a3j9d8?T19-8)L)tRaAlJdp?fjwBJnoM$#Sh+Cl9OT0Tv_iP}BX?xAE4wI7rJ3{(d{ zqYZuw6a8^~oL4@{#qn`m$QM!DfwY)5#grG*rn9e&Dhqan1Y<{=a?%P~R?xD7mcwaT zN$pJ1S+to&n_0A(0~^dnMw?5UI@m1Fts}pR_N!>WiuP-0Ur+5u(gxZz(58Vlx3k^N zjJcUMjifthxr3HFX!$gwHBq}8a_oJy*+ZK>wE39)XP~GaEu9qBEroSUVcp1g0L3aq zZ80Uq)OIG{mD;W;Qu!e%vS(G$vVz(QS`H^ai}G16&zVDhE+}RWBhZWD4#`s4(VL_ z&ZTc1=_=Z+qIMO1*O0HLwt?CPN*buWo&07{)PXjQq&sN2gW4Ume42a{wR@=DL&+X$ zKPLYfDCP!jR4VtZRMsSwHA!Vn$QM!DfwUMl;b3RVyMm$`ln)`TfEFt`wZlnglFp)h z7Wp}(t6VCrtRY`dn+Ea?Z2flfPo~P4?jZj(=^j|(%!K^Mq&mR<2iSl77Xjg*m$U=r z#grFQ(wTG!qu9T){}0e zeFOPM(kCg|LB5Gp;g>O%=GLW4pNlf2#qtc1RFc+_){{1pHjxg=l>M|aQ|zD2l%6+{ z-%Va;iKHk?v>nKogU9-uC6QN9Qcr#(`9|_hr1;Hl*c6dAh9p80sR~Pk@-X{GT1Q$> z+DO_&S`?A`6h*|coV1>jMwd!Vl`S^qq?M#a_&F!!o&VyTNX&ZDM$#tA zo5-tN@m2VKfY!+ukuN7-PQIReJ^4n`oq2NJvNKQ4&{V!e@Z^iclP{}%5hX?W>>Ft% zC6(ms$k&mtCtsg0t#5+w=7L7@JIObZ-%DN!^#=yBfR>ZX#`h1T(Web1&lPxKt#SJARcmrrmq_cI%49xlDwE3ZER#M|_7I!O9*j>~PufU%Bl#xsP2^QiwnbV_ zI;^L}9M)4}R#Gy%r$nwJUr*Xd+C-{)NrduV%#?g3X&q@j<@Mwn$v2X3B2~TFI%zp+ zC21XLJ!vCp6RD?ZFaq(yz1De17j5_wo(iCjrZJ!xZKnGcQRo5(kjSN+&lKhYMEFDGA4 zzLI<;`8v{i(niv~{iLnE{Wz2RGY)AvX&q^Oe~DaAzL9(*`6f~|K(uNAeaRP*FDGA4 zzLK<#boBsfarFRcv7VAm1Ektb8z}MX$k&svC*MfgL`f5QHHbYK#0cbz z$afkfEp{3tEtXR>Mnu zG*Pm5u(YC%VjR+P(n`|$qa<=Y`9{(vQgyV*i(D$cyDAgb>!KWjmsULG(n=8}oyw(^a`Kg=b)@yA zjigPaY6#mUtsEjTE6LZ9){|};BJnp3k@$_Y**QewH<53mjT*{0q~)ZQq;;hAq>ZFa zq^d&f%PXYCa`M9}q?KV6(n=*Ivn!;PI`Z|jY;>vkHc_sIv0c(~(n``g(t6TH(k9Zq z!=#nH!=x3@aEV_uT>4Nnoc$rKB(0;o-ld{#q`Zk#jbOW^<)oFQb)>6DNGq#HNGtV} zY#JeLHIi4ya_o*}1k!TSO3Le8D%yI=8%di;)ku1gb{Z+IbQ&qGlv6Tnq_k2=zK*n> zv}vTY*hF59Vg%CiQ6jH&smSXnuP1FJ-8o8H**QvDX`*EBC}~BFW*pLT(n``g(#Fve zxsiMmsTxE3F(NN_sYog*pFKuenLS2YsiS1|7-^-Rd?RTSsT#{Tq~)ZQr1fJZay|J* z(k4=MoX9=LNqo<762FL&qT?h^r{koRa`Kg=b)@yAjigPaY8=}oEhnuTCz0#O*OP7< zC#`H6C#^J6vU8loZ=$6-UhLKJj6+&ZT1i?*T2Ifn#^`d%SkIq>qzTK8%foP9G4SmPg+h|NqL=1MO#mKBWV+Nw|(605$^U#ciY$9 z_H(!W-R%H(JJ8(@a<_xs?NRRbXm@*zyDfLOL)`69cRS484tKXB-0iXMR(_a8@*eAM zk8`)DsrJqi=XvK9XOHuvlj%9qQ|UR!bD`%k&ugA9J^MYG-VWX)yaT=E-dnu4d7t*a z>V4PyjrSs7z3*{f*OVbC<5L!-T$XZW%8eSH{f_?V!&suNVHny9+tu3Ha0ThR;8R`kKM z6-VO!TR%KoF+fdIgVf3R9}K3eWAJPRe!m}gyiUav9F=%_V+DM#=TzO`H ze~!?K+{?)eZB4!~d^NNWuO z{4s5%CX3rXk!>))9Az-WyqI|bP=i^P0-6DtQJI)`cn<=3}#$=&@#+MgQt{=K?lLgU?v?7Iu^6iV9u3- zPRGnNcq*s|=vkPZ22a-XLHrv*4gLeoBf)QzXOZxKfAjIslM1!~mGdLrm6dNL%hg5v!qJp~m1y9Xpqpho>wpA7mHUdYz!Z=go~pic$;QO|^A zAE;4jW)^7B%z-2w)TlP*4A8b_E+mDZMs+o3ftHxFAt?nls=tYW4lw6HG7!|LV@);a zNOLYEqd<*1(JTg?VwONM71XFXrUvwMQwzx%phhh+%R$dIDxS;`k`!0cQn4&AMN-n(qP|FK6?cKr}~k54YG9M$!ODKNd-Wpc@ou z-P#6hr0DkdJNLeMj~r?(C;g{rhMarPJ?Gr7bI(2Z+&lNp`~!sl!F~nff5jTdXOnjj z{*3w_!X25dyqbBy=>jHzGQvH`j+)=>pRwWtv|8;%zD%Mp7kS3*;)I5{Vw}`_6P05_DOru zK4&l6AGaO5Zr`xC?4P%P!G6j9y#1>EMf*4H-?#tB{;&2AaK|B$IFMLL{QbnQCH`IF z&BPBAcJkrmKTLi(`6tOksnt{`^@Y@Lr2ZoH3+dlTzn=bndOCA5^IT><^UpK?HnT5# zF?%(;k^M&Y_p;y3{#Di<+BbA)=&_+c8cKG#GT|y=|7GE66&twl_3q){!81b&ckRYu zsUN{zyCP)sKHRl?KP>j6SPLG-TzeEV?NQvf`x&eW6Ic@_uqI4kO_;!%Fo88;0&Btq z)`SVH2@_ZoCa@+5ofVsTb7y-%I)n zSc94WmPy|-=}(#Tr%d{`DgR$etkic5{X0T`7xX~a|9unx(D41GMf?HWH)D)n6tia& zlvcBOXn@ zrZ!UlMQx_Pue#~i!1JcHkvfF;JMn%^eJt}V;-dZKY|;KF**C3j`c3QWL$6ss8hR&S z3%{XcsV2OEFXP>WNAL!$a}!qk4Or_pV5{GNrG5h~|1#c9*yuN4lbeuu6ZW_XYy3q> z_20uY_#(S?V!qLCxhEe}Cr(z%<%PQ6X*k>S4afIqX3EpR9xs=lB5=kH<{BIBn&;Fj zE889S)Rkh<)y_hG4eM$LJivklUTI!k;?oj!I%mGEB0JFMP- zsGh{b$J8p`A<=*!0xHcM1(iXp6+j=JnMEc98*WfuT{%6gNcmpJy4i5? zt4|%L;s*7myKqXao-bW3FJ8Pbe>RSuJheQxa$YTO`+?h>EL~6^U%Y$)^Bw}c{47O? z5JE(VJ`p1!`bGHgF;XRDo0-rNn<^xUN&+)D?=)RRJ9HIkNED&|#X9z>DeMklZ7)l6TC@Vh%h(UKWMgWmwh(RCAI@hryCKmz8 z6=KkLjY0P{j{w4R_?Rh~fRr3!ARki>WMh~F60Sauk~4e|URW3*U0R5v0vNuKE95ch zRm3-`zU8timpS_7>Za#Vd&|vw#cTT@u+SBfL_!)O4nv_)0uZBzMI%F)c<8F@p&rxJ zeu^}XpvG%R%Gqq1LI?!4y02Sd22|7f3FeT&)OZ6DyRarrMno-J4ytQ1DFlF>3{`o) z-DtQKcG{mj<3fk(73uUs-Pvfh{h(ecRaG^fVBzzdo(G*;a#{`y(n`&P?Ev42lZ|q@ z-l_+6r_o2k+O%{H;HbnCoYiWsT2*4Y{MFV?@U^}A3oeSHTAsXKE}wEL*YOLTr|WJ5 zSZd@TmRdQ8#Z24-OYI!wrG^e-F?098Qd9TfrD6`EMPCQe9Fx7c_ePiYXPgGP{O?kL9Jb-DIKJp zcAa3;bN41*#Jb#cF>CiGpW3W9s;(#g!+?HXc2RCi)}z5%m&B^Lm;+URx!$U{tF8Lh zVy9iH4f2+Jy?8o}LE5Uf2QpsxV3sAfx#oKQAYsnifa}o;_8_5}(`~Qm1cSMjw_Aa; zHGrS%blg@|#hY@`^_Hc_YNcMe?jsqas+Jr0fzzrwURABMVVkhV!Fj8=Y91yfa7!(c zk)&QP80##eQ+|8XBcXlP@oV$#s;kbpK_8}(GMb^9T0`PK7rdpu$EDl|EYW#v`}DoU z=U8ZO3J4zv4d9TUMQcGsT}++!+D(I@F+jv*ZDY2d6Po9W*J=(pQ+~IJX#CovPUH++y+t~2j z4Gg1kLG~!oiM!(%Q!QlUszJaah1iT!Ytih(2yMWn)l*Rg^VTrNY^0G41C;W$Cp8hufzIp4zQ%A?B3X{dwVg+a*HZsc^9`Tjm@g46;*n^jW)x$MTPgFRT{m1 zN-el+n;RR{+XxqKHzLfgzDIE5msBN^X-65kI=evu6MQ$Nw*^W4)$kjaZ=c! z(-?^>7%E}Yk=>wUf(cw zYmmI)`W3G(8>WaDDva)3gpWKmGan+7tk<1(gUX~Ab#0Wqh(&L8O4>JrhZQ4X#Fm|P zH`u<6&5$2u5i4ebG3ZGkQn~CloGpQ3^ko?SogIx4!|^fh zL)4;>FyHQMd-aW45RsOz*E<}ChR;4m`1^~-EPN4bIA1Vm+#W{+EmFPaMOd-T;>Sd| z5;~=K8scGyag2E!%8R@Y zo)xULBQ~szI&qU*9#!(s(=yj@!e&AnP{%qpL0k!uPn~WcSZr0=RV-+sQ8mq-G!ums zvti)M?ZwjE?Zb z!H285yt%$!uhd~M7q{HXCO2%b%9ZO1W}@wFFFBrHa~cY}r;R$SnG_WHkkYAGGQRZ7 z3}KdV$j;%g39BED6c9sQaHe1Yv%FGcao~piAJ(XbIgnTl>HwpI6ys8aT2pN_)YWD~ z4lSf{@GhlQjCCKN7_QUQEypQ~(K~e+9#^}mC`_Jp$oZ4_Yb-%& zO1T`KFT!~-YIeeApR2cSsQH@X!S{t%pdB+0N%(&-285cDU7dnk zMp&B#v$Tb79@saih8tjL2Q`GQ<2CAt<-AKP5;h&4Dnky4XduNVn|1>u6^h5}spol+ zN?ZUVf78XetS4ohYd_C28WIgE@m@Y5z2jy0FdR|j4L20CK)iN|7^--4&^qvblL@=;HGSUeri`8q~64V=gNuc0+z1VL6$zki0$ zCDK=ony);(MUhXOJc4sX9N2}OiHUlW?))6A?6wcOUKHBv!A>-#DZ6AIifKTfj2Q#D zxD~i9x}g0UVk*|R(zeB~xX0u*3V7R`Tq%1@}rnE^@fc;a8aCI>>P8tJ-K1D3a4^m8$b;OdPk(M6G zaL8vYEwS9{!rCWrW~r8Ow2sI)tk{iJFi0Y!P#0n&)i(l|u#=ZvI0`E4m-ZV}TigfL zaMuGRel8b)QnS;52N1w_qat`vCc3)#j0pjIzTmDqn~eaQRc$rZ+^+pMmd=}XuifId zt3M=rHPHBV@rFj$%AEb~l`phY?AY6|y>0RA$=l?OR}p7DadM_yuIOWF)oDYkMJDt7 z_=>tkW3T2NAL65X3NHXbz17(a)Dm{tobX|fRtVq3=VVu~wNc*lQjqrkl`jWhm_{4! zma5^cmo05bWZ{8P(eW8Kz|g>lcITl{8*tJ%ilGL{h_)~rv3>;Gs)UVgK%gE1i_H#_ z>d*>4iQd4Ui%+2!@jW-2`2164it3a2EL=^^;eNn|;9SrG;F?JJ0<9rW5xI-_to*Q= zQMiFNauewWINYBi?+osx;5OR>*2TM5Qr83XoAMJ+D$7zId9WVMt2X-ELO&|#D;#CO zyu+oQez5%H)e@^CKji$OZG`-L5})%gA;hO1xKXfz_gVb?Eacim=;Ln^_W+{5k16Xz zOK3p_BhdzxEn;h`fR}-9A?@P!K@0E}Mz8~F1>gQq#GL|+Aa04M_u?~m^&&>}Fv3a1 zGpK0+GJjF2NA6d)$$w~BN~3mcVq{(1Yzve+xPY4qT1EkITf~hJXL22#6P%d-)B5@PHC-;{6@;TFEPN~QiD>QE_;h#Hwh9g|AnNAY2%(ZHRvqYQ~wLBo$K zyqh6Y@fjeLk1^zY;>gj8aQr*KHGvB?93M3zrS-auere4vBS)y)Qr6*kS!K~rYD_p= zcbAEF!$MrJeb=(qCi4Q7;pY&p?-;SYW(M?bv(omn0 zzlieMA`{v#aJ+LTrq8*b_Kb4dCfC6z)#pIj@DgN1Hpl~u5T*NEBd1Vb8|?;1K7{+! zObM$4DEk>l*j0mN+{X@aH91D^hcpyXX!|uqLlLEfdyX?E521*%+NWqJqLfh95tD~d zL|N@qG}Nbzgex6;8Nv}YT$@M@;fPv7ySUQnyc&+Et1)T__o+jt`@PUmoD%lpw6r>+ zgw|DMwBD$U))Mv5PF+wV(WtBa@zGU7;Yg|!M^2?UQVOHVaZnjO3QSYxOzT)nmT9(4 z$%x9DJrqiXII}JTXP}Jkn?fx46x86wva*SyvQhx@ zc_wqX2WI6E0@9<34Bl=RF^cVWvzoh)xNR72hlCJuCq&#~OUH5#v6YAN-Cwu!-8=cNW-D|* zBl!!x2@^k|vw<+eno-+l(Kisg|Hv9knab{lW#7#ftwQ%+q5JYjO!42yX2{xF%b3O7(!;RdIJZQul| z$vvcOTh9m`4uP&AHsHs|kE^f(N!tpSP(YR?+jikn;Sx&Dqoeumr#Xu-;U2QG`0Z<| z?x(GkgySkXEK^KS2NY?)h{g%=pt95UNHL+2Dceq=z-}**v4+{NA6p}tqHQ09Ojr~2 zKx(S7L@EUs@(ofjFhXgNA{gk1U^>KfatF{YrpMKRag_xQVa7Cb1P#{UxJnFLDF_!uI6**9k*~LyWo$tPqE`spNuuxvbRL*x_MH)@e zs8?Y0&{7PKqSvk!~ zA`M|2uy7d~9YPY z-fW?FqR@M$&|4_<&K7#-3cZVk-b$f&wa|O6(7RUXRSUgZq1Pz%I%Aumw?tKOvVlcjHv&r3iv@L=)CcHNbgWnCQTS61CAP|f|2qx3e70fSmgm#kc;e@0)WKV~vUk6q8pvBxjlHB{XW(kKVEENz{VLX&7 z^gaw&W;hVJ{m2hS8c&o#)hCFudPymZ0#Xn_FN>IM`3q62`S_*!b0ztMR{^LJg(?CYOA_|_jh`JXO5tesu$#%^Lm6=2IXPUH>j zmg?Ba@GN5qVH5i-?gc^SQ3f}nJYeC0jf+P?xMgq3hH`X4A{Wno;Nv~)l=dWs+r&8W z;YH-&`N0n48EHs$2QvFnRLygY_nemuY#6u1@r=Xl1m4}8U{N(HfpCw66t)y zP7foJHS0DPt~(5VnRDhIq5s9;&k*z;gLh^0oXH}CQJM03g<$$lDov$Efm=v+|0Ap- zr?*yPG8|yRFQ?Edg|E`ab3A;y2U`*bAtFX2!-&O241D0B&r-OAWmApGhQZDeV~OhF zr6yTw`I=;@2^+&Kd>9r6X2EIhj^H}^j^H}`Xxiz3OFo0z``xDQ7DWV zm{BVpFa(y4h$asA)43#eqPL{dTT%gh7O8+C+zsQ5puiFIrQ6l{Y1O)22nKEKc8A#5 zZWsSR5XzGL3ic`jV7Yun#PLd)yd%jwVe(Z;YCncMoDqdWkPH8JbQQOC6~qAnWAT!X zKBp_Z&9)xkL%ztdUy6l4q{~NvuS$Uc_^(SMr1O%CmR8sWV5{-gI)4#+6ntg#N3rL|@!ft6&6)+5tgUBtDgiv_1C+`O@m2e8x>ZF?+* zJsc`x%)uH<<)`UW%!>WfjsX=63(!5?ue-XVcQHMr8}~rbO(k?HYU*Ei8vgblMskb4ADV`WRxL^Bs~#RSsKY~l*}SI93_WYSQY88 zbl3xITw;hCGVF^o>_c*Yl-y4l=xv}NA(tqxc$gUauqipsOt%HdP+;1RNy4cfWsplF zfWN|aY(-^ZM-RlU7~+;}FuK5)b(=FBJl)%I)NMiCiK95fI3q!UlAF0F-pd3iaAz4o zi6>B_bwJNZ7Axj}w71?Lzcijx3b^q*fwZj_sq zxxhWZtS*%iImDD?SvD7&G;BViA;}{UXFK-|Cv-MO$aVt7TL&Tr>?)qo42;0=3_3+6 zmx98n$eqW6W@7}l1?d#%YzEdjtQ8y&3SvvQ7(11-ItwFYk?253#xCZZCNl#0oX&== z%|(r81pGyf$9`u;;~4?JqVd=SUDbF-z+csP?2Mk%ct*fKr}5Y>UDJ3*z+cmN?4ha} z&j|Rc#$#hu(|AU}*EAlxt%k-k0=}W~*n)BTLH3M*?`S;sWr4;s0zS}qY}U3ko)Pdy zeAvO=(s)L|-_rcp)^#1TQ+YQf&n(3-PZ6vK zEtSKQtd;A$DqTG1tocgq!xv}Z0_F)Sao~sbl<9R;+m< z&UeQUJMRZ7^$~ns03QBm<2naED#tC3Cu%{^@qcz|YNH<1HrFOA?dDV^6c3)G^`~N` z!AUmGCdOpaQXh=bOikw26mDK!n|VC;jBeUy`2o?#yrf{9c)@-E8|F zpn=eIapDzP`V9C7EBg%S571I}Xjf(w`QOq5SP1yrreQI$?;aEjY(>NN6NU@+fR77ti7J8HtQm7qmhAI+W718^1|}J`n%tF>+^ql;-kO#JOBBYepGl`HjPs?JPtGk z+gF{UMT{o@ldu9#qgIU@09<6lJxF(|g9o6OBaJzcS7}u%TlV~u-@sf8Ms>u!>q+LEc2YD;g=zsICO`|E8pG!>t@n2o~#Md6l zz4$HshD^Voj}R{QlRJQXj8CcZeA`=SG?tutOEvLKjpw?PyghARZ+!s8>=t22{C_;= zcVh7SGE{eGN3{LHLNtIU-&wq`K4YnCu!G_2S`w?Ix}uixsqq!uZMh7ng!7^ENS6?v z2F%y*Cja~YruD&gUD9F9Z{6aHMN-e>uLUG5l-4%Tr##MfhzYXwxE z?ckEAoKERGO2VMNlKeIg@6o2{llsAu?h4N|?!)N6BRCI(4AdXuyGro8Yp9KPlJp%8 zxC*#yQbVeUQhBPxTSoy}M~(&DMk&T_q!b|!-f0NwylundHTEPbv55OC7tu4`!(rce zLngY_!kZHM5R5l9>i8~`=x)apPVjhtUn9&b>uz=Iv#Q-8Q^q=;eyo8`zm$xioN8?9bbFPW-`{V$z8Hh zkU6&S{j#q}uXqd@mU$R`IEweQyxEp<@JBaST4-k@)|;Qc4U>?k+#W&=)X;Ni1Mh7$ zL<>5Q6??jY@ek0)8MI`EpX=jQm9|_kbV{AMLOC(T@&4<`C2G?wzOTiCro?Zw#K&&$ z{wnn;d>hv8e7mI^$@X!4GZOFCO=HZf>KIZ_;e8D0BZwWm*Dwl40aXB3z;je^ltmS{ zB4!ak4W6fgcLhiJ%*xo#L5rxZJaOjvaS^jypFD@79JPbv{I{?FYZ540&>p3Z@M{(S K_Vxdr1pW`7USj$H diff --git a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/lib/netstandard2.1/SpacetimeDB.BSATN.Runtime.dll.meta b/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/lib/netstandard2.1/SpacetimeDB.BSATN.Runtime.dll.meta deleted file mode 100644 index de2769892f6..00000000000 --- a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/lib/netstandard2.1/SpacetimeDB.BSATN.Runtime.dll.meta +++ /dev/null @@ -1,33 +0,0 @@ -fileFormatVersion: 2 -guid: 8ab8f6f35b91340e6b1269b6448e3bf0 -PluginImporter: - externalObjects: {} - serializedVersion: 2 - iconMap: {} - executionOrder: {} - defineConstraints: [] - isPreloaded: 0 - isOverridable: 1 - isExplicitlyReferenced: 0 - validateReferences: 1 - platformData: - - first: - Any: - second: - enabled: 1 - settings: {} - - first: - Editor: Editor - second: - enabled: 0 - settings: - DefaultValueInitialized: true - - first: - Windows Store Apps: WindowsStoreApps - second: - enabled: 0 - settings: - CPU: AnyCPU - userData: - assetBundleName: - assetBundleVariant: From 7f6fd180181ae293d2b012f49490681310c20e70 Mon Sep 17 00:00:00 2001 From: Zeke Foppa <196249+bfops@users.noreply.github.com> Date: Fri, 23 Jan 2026 14:58:10 -0800 Subject: [PATCH 3/4] Re-run the LLM benchmarks update (#4110) # Description of Changes Try fixing this again? It seems to pass on PRs if re-run. # API and ABI breaking changes None. # Expected complexity level and risk 1 # Testing - [x] It passes on this PR now :shrug: --------- Co-authored-by: Zeke Foppa Co-authored-by: clockwork-labs-bot --- docs/llms/docs-benchmark-analysis.md | 371 ++---- docs/llms/docs-benchmark-comment.md | 18 +- docs/llms/docs-benchmark-details.json | 1562 ++++++++++++------------- docs/llms/docs-benchmark-summary.json | 66 +- 4 files changed, 915 insertions(+), 1102 deletions(-) diff --git a/docs/llms/docs-benchmark-analysis.md b/docs/llms/docs-benchmark-analysis.md index 44f3a564731..17b5c26d78c 100644 --- a/docs/llms/docs-benchmark-analysis.md +++ b/docs/llms/docs-benchmark-analysis.md @@ -4,20 +4,21 @@ Generated from: `/__w/SpacetimeDB/SpacetimeDB/tools/xtask-llm-benchmark/../../do ## Summary -- **Total failures analyzed**: 35 +- **Total failures analyzed**: 36 --- -# Analysis of SpacetimeDB Benchmark Failures +# SpacetimeDB Benchmark Failures Analysis -Here's an overview of the SpacetimeDB benchmark failures organized by programming language and mode, providing actionable insights for documentation updates. +This document analyzes test failures in the SpacetimeDB benchmark organized by language and mode. For each failure, we provide the generated code, the expected code, the error message, and a detailed explanation along with actionable recommendations. -## Rust / rustdoc_json Failures (9 Total) +## Rust / rustdoc_json Failures (8 total) -### Compile/Publish Errors (3 failures) +### Compile/Publish Errors (2 failures) -#### t_002_scheduled_table -- **Generated Code**: +#### t_002_scheduled_table & t_017_scheduled_columns + +1. **The generated code**: ```rust use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt}; @@ -28,8 +29,22 @@ Here's an overview of the SpacetimeDB benchmark failures organized by programmin scheduled_id: u64, scheduled_at: ScheduleAt, } + + #[reducer(init)] + pub fn init(ctx: &ReducerContext) { + if ctx.db.tick_timer().count() == 0 { + ctx.db.tick_timer().insert(TickTimer { + scheduled_id: 0, + scheduled_at: ScheduleAt::repeat(std::time::Duration::from_micros(50_000)), + }); + } + } + + #[reducer] + pub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {} ``` -- **Golden Example**: + +2. **The expected code**: ```rust use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; use std::time::Duration; @@ -41,99 +56,55 @@ Here's an overview of the SpacetimeDB benchmark failures organized by programmin pub scheduled_id: u64, pub scheduled_at: ScheduleAt, } - ``` -- **Error**: `publish_error: spacetime publish failed (exit=1)` -- **Difference**: The LLM used `schedule(reducer = tick, column = scheduled_at)` instead of `scheduled(tick)`. -- **Root Cause**: The reducer scheduling syntax was incorrectly formatted. -- **Recommendation**: Update the documentation to clarify the correct syntax for scheduled attributes. - ---- - -#### t_015_product_type_columns -- **Generated Code**: - ```rust - use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType}; - #[derive(SpacetimeType)] - pub struct Address { - street: String, - zip: i32, - } - ``` -- **Golden Example**: - ```rust - use spacetimedb::{reducer, table, ReducerContext, SpacetimeType, Table}; + #[reducer] + pub fn tick(_ctx: &ReducerContext, _schedule: TickTimer) {} - #[derive(SpacetimeType, Clone, Debug)] - pub struct Address { - pub street: String, - pub zip: i32, + #[reducer(init)] + pub fn init(ctx: &ReducerContext) { + let every_50ms: ScheduleAt = Duration::from_millis(50).into(); + ctx.db.tick_timer().insert(TickTimer { + scheduled_id: 0, + scheduled_at: every_50ms, + }); } ``` -- **Error**: `publish_error: spacetime publish failed (exit=1)` -- **Difference**: Missing `pub` modifiers on struct fields and missing `Clone` and `Debug` traits. -- **Root Cause**: The documentation didn't clearly indicate the necessary visibility modifiers and traits for SpacetimeDB types. -- **Recommendation**: Enhance examples in the documentation to include field visibility and common traits for struct definitions. ---- +3. **The error**: `publish_error: spacetime publish failed (exit=1)` -#### t_017_scheduled_columns -- **Generated Code**: - ```rust - use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt}; +4. **Explain the difference**: + - Incorrect `#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]` should use `#[table(name = tick_timer, scheduled(tick))]` + - Use of `ScheduleAt::repeat` is incorrect; it should use a proper time duration constructor. - #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))] - pub struct TickTimer { - #[primary_key] - #[auto_inc] - scheduled_id: u64, - scheduled_at: ScheduleAt, - } - ``` -- **Golden Example**: - ```rust - use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; - use std::time::Duration; +5. **Root cause**: The documentation lacks clarity on scheduling syntax and constructors for time intervals in scheduled tasks. - #[table(name = tick_timer, scheduled(tick))] - pub struct TickTimer { - #[primary_key] - #[auto_inc] - pub scheduled_id: u64, - pub scheduled_at: ScheduleAt, - } - ``` -- **Error**: `publish_error: spacetime publish failed (exit=1)` -- **Difference**: Same scheduling error as in `t_002_scheduled_table`. -- **Root Cause**: Similar to above, incorrect usage of scheduling syntax. -- **Recommendation**: Provide explicit examples illustrating scheduled and unscheduled table definitions. +6. **Recommendation**: Update documentation to emphasize using `scheduled(tick)` and correct constructors for `ScheduleAt` using `Duration::from_millis`. --- ### Other Failures (6 failures) -#### t_003_struct_in_table -- **Generated Code**: +#### t_003_struct_in_table, t_004_insert, t_007_crud, t_011_helper_function, t_016_sum_type_columns + +1. **The generated code** (e.g., for `t_003`): ```rust - use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType}; + use spacetimedb::{ReducerContext, Table, UniqueColumn, SpacetimeType}; - #[derive(SpacetimeType)] + #[derive(SpacetimeType, Clone)] pub struct Position { - x: i32, - y: i32, + pub x: i32, + pub y: i32, } - #[table(name = entity)] + #[spacetimedb::table(name = entity)] pub struct Entity { #[primary_key] - id: i32, - pos: Position, + pub id: i32, + pub pos: Position, } - - #[reducer(init)] - pub fn init(_ctx: &ReducerContext) {} ``` -- **Golden Example**: + +2. **The expected code**: ```rust use spacetimedb::{table, SpacetimeType}; @@ -142,6 +113,7 @@ Here's an overview of the SpacetimeDB benchmark failures organized by programmin pub x: i32, pub y: i32, } + #[table(name = entity)] pub struct Entity { #[primary_key] @@ -149,227 +121,134 @@ Here's an overview of the SpacetimeDB benchmark failures organized by programmin pub pos: Position, } ``` -- **Error**: `schema_parity: reducers differ - expected [], got ["add_entity()", "init()"]` -- **Difference**: Missing `pub` modifiers led to visibility issues; `add_entity` reducer was not included. -- **Root Cause**: The documentation should indicate proper visibility for struct members. -- **Recommendation**: Include visibility guidance in struct and reducer examples to prevent visibility mismatches. + +3. **The error**: `schema_parity: reducers differ - expected [], got [...]` + +4. **Explain the difference**: Missing `pub` for fields in structs which are not public, causing access issues. + +5. **Root cause**: Insufficient detail in documentation about struct visibility and reducing/scheduling attributes. + +6. **Recommendation**: Clarify that public fields are required for structs defining database tables. --- -#### t_004_insert -- **Generated Code**: - ```rust - use spacetimedb::{table, reducer, ReducerContext, Table}; +#### Additional Observations: - #[table(name = user)] - pub struct User { - #[primary_key] - id: i32, - name: String, - age: i32, - active: bool, - } +- The focus must be on both visibility modifiers and correct API signatures for reducers and tables. +- The need for `Result<(), String>` in reducer functions is missing in many generated snippets. - #[reducer] - pub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) { - ctx.db.user().insert(User { id, name, age, active }); - } - ``` -- **Golden Example**: +--- + +## Rust / docs Failures (22 total) + +### Other Failures (22 failures) + +#### t_000_empty_reducers, t_001_basic_tables, t_002_scheduled_table, t_004_insert + +1. **The generated code** (for `t_000_empty_reducers`): ```rust - use spacetimedb::{reducer, table, ReducerContext, Table}; + use spacetimedb::ReducerContext; - #[table(name = user)] - pub struct User { - #[primary_key] - pub id: i32, - pub name: String, - pub age: i32, - pub active: bool, - } + #[spacetimedb::reducer] + pub fn empty_reducer_no_args(_ctx: &ReducerContext) {} + ``` + +2. **The expected code**: + ```rust + use spacetimedb::{reducer, ReducerContext}; #[reducer] - pub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) -> Result<(), String> { - ctx.db.user().insert(User { id, name, age, active }); + pub fn empty_reducer_no_args(ctx: &ReducerContext) -> Result<(), String> { Ok(()) } ``` -- **Error**: `data_parity_insert_user: spacetime sql failed: Error: no such table: user` -- **Difference**: Missing `pub` modifiers on struct fields and the reducer function lacked error handling. -- **Root Cause**: The need for visibility on struct fields and proper error handling for database operations is not emphasized in documentation. -- **Recommendation**: Emphasize that all struct fields must be public and demonstrate typical error handling in reducers. + +3. **The error**: `schema_parity: describe failed: WARNING: This command is UNSTABLE` + +4. **Explain the difference**: Missing return type `Result<(), String>` for all reducer functions causes the failure. + +5. **Root cause**: Documentation does not clearly specify that all reducer functions must return a `Result`. + +6. **Recommendation**: Update the documentation to explicitly require a `Result` return type for all reducer functions to avoid compilation errors. --- -## C# / docs Failures (4 Total) +## C# / docs Failures (6 total) + +### Other Failures (6 failures) -### Other Failures (4 failures) +#### t_008_index_lookup, t_013_spacetime_sum_type -#### t_014_elementary_columns -- **Generated Code**: +1. **The generated code** (for `t_008_index_lookup`): ```csharp using SpacetimeDB; public static partial class Module { - [SpacetimeDB.Table(Name = "Primitive", Public = true)] - public partial struct Primitive + [SpacetimeDB.Table(Name = "User")] + public partial struct User { [SpacetimeDB.PrimaryKey] public int Id; - public int Count; - public long Total; - public float Price; - public double Ratio; - public bool Active; public string Name; + public int Age; + public bool Active; } [SpacetimeDB.Reducer] - public static void Seed(ReducerContext ctx) + public static void LookupUserName(ReducerContext ctx, int id) { - ctx.Db.Primitive.Insert(new Primitive + var user = ctx.Db.User.Id.Find(id); + if (user != null) { - Id = 1, - Count = 2, - Total = 3000000000L, - Price = 1.5f, - Ratio = 2.25, - Active = true, - Name = "Alice" - }); + ctx.Db.Result.Insert(new Result + { + Id = user.Id, + Name = user.Name + }); + } } } ``` -- **Golden Example**: + +2. **The expected code**: ```csharp using SpacetimeDB; public static partial class Module { - [Table(Name = "Primitive")] - public partial struct Primitive + [Table(Name = "User")] + public partial struct User { [PrimaryKey] public int Id; - public int Count; - public long Total; - public float Price; - public double Ratio; - public bool Active; public string Name; + public int Age; + public bool Active; } [Reducer] - public static void Seed(ReducerContext ctx) + public static void LookupUserName(ReducerContext ctx, int id) { - ctx.Db.Primitive.Insert(new Primitive { - Id = 1, - Count = 2, - Total = 3000000000, - Price = 1.5f, - Ratio = 2.25, - Active = true, - Name = "Alice" - }); + var u = ctx.Db.User.Id.Find(id); + if (u.HasValue) + { + var row = u.Value; + ctx.Db.Result.Insert(new Result { Id = row.Id, Name = row.Name }); + } } } ``` -- **Error**: `spacetime sql failed: Error: no such table: primitive` -- **Difference**: The `Public` attribute is used incorrectly, which may lead to schema mismatch or missing construct. -- **Root Cause**: The correct usage of attributes may not be clearly noted in the documentation. -- **Recommendation**: Validate the appropriate usage of attributes in table and struct definitions. ---- +3. **The error**: `publish_error: spacetime build (csharp) failed (exit=1)` -#### t_016_sum_type_columns -- **Generated Code**: - ```csharp - using SpacetimeDB; +4. **Explain the difference**: Use of `user != null` instead of checking `u.HasValue`, which is necessary for nullable types. - public static partial class Module - { - [SpacetimeDB.Table(Name = "Drawing", Public = true)] - public partial struct Drawing - { - [SpacetimeDB.PrimaryKey] - public int Id; - public Shape A; - public Shape B; - } - - // ... additional definitions ... - } - ``` -- **Golden Example**: - ```csharp - using SpacetimeDB; +5. **Root cause**: Lacking examples for nullable types or option types in the given context. - public static partial class Module - { - [Table(Name = "Drawing")] - public partial struct Drawing - { - [PrimaryKey] public int Id; - public Shape A; - public Shape B; - } - - // ... additional definitions ... - } - ``` -- **Error**: `spacetime sql failed: Error: no such table: drawings` -- **Difference**: Similar to the previous error, the `Public` attribute on a table may not be necessary and may lead to runtime issues. -- **Root Cause**: Documentation isn't explicit about when to use visibility modifiers and how they interact with SpacetimeDB configuration. -- **Recommendation**: Clarify the usage of attribute visibility and provide clear examples. +6. **Recommendation**: Address nullable type usage in the documentation, emphasizing how to correctly check for value presence. --- -#### t_017_scheduled_columns -- **Generated Code**: - ```csharp - using System; - using SpacetimeDB; - - public static partial class Module - { - [Table(Name = "TickTimer", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))] - public partial struct TickTimer - { - [PrimaryKey, AutoInc] - public ulong ScheduledId; - public ScheduleAt ScheduledAt; - } - - [Reducer] - public static void Tick(ReducerContext ctx, TickTimer _timer) { } - - // ... init reducer ... - } - ``` -- **Golden Example**: - ```csharp - using SpacetimeDB; - - public static partial class Module - { - [Table(Name = "TickTimer", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))] - public partial struct TickTimer - { - [PrimaryKey, AutoInc] public ulong ScheduledId; - public ScheduleAt ScheduledAt; - } - - [Reducer] - public static void Tick(ReducerContext ctx, TickTimer schedule) { } - - // ... init reducer with more clarity ... - } - ``` -- **Error**: `spacetime sql failed: Error: no such table: tick_timer` -- **Difference**: Incorrect usage of parameter names in the reducer; `_timer` should just use a clear name. -- **Root Cause**: The documentation should illustrate effective naming conventions for function parameters. -- **Recommendation**: Include best practices for naming and structuring reducers alongside examples of scheduled fields. - ---- +### Final Thoughts -This analytical report outlines the specific failures within SpacetimeDB benchmarks, pinpointing concrete documentation enhancements to improve understanding and usability. By implementing these changes, developers can reduce occurrence and severity of future benchmark failures. +A thorough review of generator patterns and failure analysis indicates that clarifying visibility, return types, syntax for scheduling, and handling nullable types are crucial improvements for development efficiency and error avoidance in SpacetimeDB. Documenting common patterns and providing clear guidelines will enhance user experience and reduce test failures. diff --git a/docs/llms/docs-benchmark-comment.md b/docs/llms/docs-benchmark-comment.md index e6e73436a0f..6a2bcda9a11 100644 --- a/docs/llms/docs-benchmark-comment.md +++ b/docs/llms/docs-benchmark-comment.md @@ -2,16 +2,16 @@ | Language | Mode | Category | Tests Passed | Task Pass % | |----------|------|----------|--------------|-------------| -| Rust | rustdoc_json | basics | 22/27 | 74.3% ⬇️ -9.0% | -| Rust | rustdoc_json | schema | 23/34 | 65.3% | -| Rust | rustdoc_json | **total** | 45/61 | **70.2%** ⬇️ -4.9% | +| Rust | rustdoc_json | basics | 22/27 | 74.3% | +| Rust | rustdoc_json | schema | 26/34 | 75.3% ⬆️ +10.0% | +| Rust | rustdoc_json | **total** | 48/61 | **74.8%** ⬆️ +4.5% | | Rust | docs | basics | 5/27 | 11.1% | -| Rust | docs | schema | 8/34 | 20.5% ⬆️ +5.0% | -| Rust | docs | **total** | 13/61 | **15.4%** ⬆️ +2.3% | -| C# | docs | basics | 27/27 | 100.0% | -| C# | docs | schema | 25/34 | 73.7% ⬆️ +10.0% | -| C# | docs | **total** | 52/61 | **88.0%** ⬆️ +4.5% | +| Rust | docs | schema | 4/30 | 12.5% ⬇️ -8.0% | +| Rust | docs | **total** | 9/57 | **11.7%** ⬇️ -3.6% | +| C# | docs | basics | 24/27 | 91.7% ⬇️ -8.3% | +| C# | docs | schema | 22/34 | 63.7% ⬇️ -10.0% | +| C# | docs | **total** | 46/61 | **78.9%** ⬇️ -9.1% | _Compared against master branch baseline_ -Generated at: 2026-01-23T12:08:22.290Z +Generated at: 2026-01-23T20:54:53.246Z diff --git a/docs/llms/docs-benchmark-details.json b/docs/llms/docs-benchmark-details.json index b6074d5fccf..69aaf946ba3 100644 --- a/docs/llms/docs-benchmark-details.json +++ b/docs/llms/docs-benchmark-details.json @@ -19,7 +19,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, _count: i32) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, _count: i32, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, _active: bool, _ratio: f32, _label: String) {}", + "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, count: i32) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, count: i32, name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, active: bool, ratio: f32, label: String) {\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-000-empty-reducers-golden", @@ -35,15 +35,15 @@ "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T11:56:37.595950645Z", - "finished_at": "2026-01-23T11:56:59.057829079Z" + "started_at": "2026-01-23T20:45:58.319507359Z", + "finished_at": "2026-01-23T20:46:21.669318285Z" }, "t_001_basic_tables": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -53,7 +53,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "use spacetimedb::table;\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", + "llm_output": "#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-001-basic-tables-golden", @@ -69,15 +69,15 @@ "llm_db": "basics-t-001-basic-tables-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T11:56:37.595256337Z", - "finished_at": "2026-01-23T11:57:01.112482005Z" + "started_at": "2026-01-23T20:45:58.318770844Z", + "finished_at": "2026-01-23T20:46:19.785596995Z" }, "t_002_scheduled_table": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -87,7 +87,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n let t = ctx.db.tick_timer();\n for row in t.iter() {\n t.scheduled_id().delete(&row.scheduled_id);\n }\n t.insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext) {\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat(std::time::Duration::from_micros(50_000)),\n });\n }\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -99,14 +99,14 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Blocking waiting for file lock on package cache\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Blocking waiting for file lock on package cache\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling find-msvc-tools v0.1.8\n Compiling either v1.15.0\n Compiling zerocopy v0.8.33\n Compiling serde v1.0.228\n Compiling shlex v1.3.0\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling anyhow v1.0.100\n Compiling nohash-hasher v0.2.0\n Compiling bytes v1.11.0\n Compiling convert_case v0.4.0\n Compiling zmij v1.0.16\n Compiling arrayvec v0.7.6\n Compiling heck v0.4.1\n Compiling humantime v2.3.0\n Compiling keccak v0.1.5\n Compiling bytemuck v1.24.0\n Compiling serde_json v1.0.149\n Compiling arrayref v0.3.9\n Compiling constant_time_eq v0.4.2\n Compiling getrandom v0.2.17\n Compiling spacetimedb-lib v1.11.1\n Compiling itoa v1.0.17\n Compiling hex v0.4.3\n Compiling smallvec v1.15.1\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling second-stack v0.3.5\n Compiling log v0.4.29\n Compiling memchr v2.7.6\n Compiling scoped-tls v1.0.1\n Compiling cc v1.2.54\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling blake3 v1.8.3\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling ethnum v1.5.2\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:18:14\n |\n18 | t.insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:20\n |\n14 | let t = ctx.db.tick_timer();\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:20:35\n |\n20 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0422, E0599.\nFor more information about an error, try `rustc --explain E0422`.\nerror: could not compile `spacetime-module` (lib) due to 4 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.33\n Compiling find-msvc-tools v0.1.8\n Compiling bitflags v2.10.0\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling thiserror v1.0.69\n Compiling arrayvec v0.7.6\n Compiling bytes v1.11.0\n Compiling humantime v2.3.0\n Compiling convert_case v0.4.0\n Compiling heck v0.4.1\n Compiling keccak v0.1.5\n Compiling zmij v1.0.16\n Compiling second-stack v0.3.5\n Compiling arrayref v0.3.9\n Compiling constant_time_eq v0.4.2\n Compiling spacetimedb-lib v1.11.1\n Compiling bytemuck v1.24.0\n Compiling smallvec v1.15.1\n Compiling itoa v1.0.17\n Compiling getrandom v0.2.17\n Compiling itertools v0.12.1\n Compiling serde_json v1.0.149\n Compiling hex v0.4.3\n Compiling memchr v2.7.6\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling cc v1.2.54\n Compiling rand_core v0.6.4\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling blake3 v1.8.3\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:15:36\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:23:44\n |\n23 | pub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:15\n |\n14 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:15:16\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:17:39\n |\n17 | scheduled_at: ScheduleAt::repeat(std::time::Duration::from_micros(50_000)),\n | ^^^^^^ variant or associated item not found in `ScheduleAt`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:23:8\n |\n 22 | #[reducer]\n | ---------- required by a bound introduced by this call\n 23 | pub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `register_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:746:81\n |\n746 | pub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl Reducer<'a, A>) {\n | ^^^^^^^^^^^^^^ required by this bound in `register_reducer`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:23:8\n |\n22 | #[reducer]\n | ---------- required by a bound introduced by this call\n23 | pub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `invoke_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:45:19\n |\n44 | pub fn invoke_reducer<'a, A: Args<'a>>(\n | -------------- required by a bound in this function\n45 | reducer: impl Reducer<'a, A>,\n | ^^^^^^^^^^^^^^ required by this bound in `invoke_reducer`\n\nSome errors have detailed explanations: E0277, E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0277`.\nerror: could not compile `spacetime-module` (lib) due to 8 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:56:37.596087756Z", - "finished_at": "2026-01-23T11:57:51.917718856Z" + "started_at": "2026-01-23T20:45:58.319658080Z", + "finished_at": "2026-01-23T20:46:56.319486952Z" }, "t_003_struct_in_table": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -116,7 +116,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}\n\n#[reducer(init)]\npub fn init(_ctx: &ReducerContext) {}\n\n#[reducer]\npub fn add_entity(ctx: &ReducerContext, id: i32, x: i32, y: i32) {\n ctx.db.entity().insert(Entity { id, pos: Position { x, y } });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table, UniqueColumn, SpacetimeType};\n\n#[derive(SpacetimeType, Clone)]\npub struct Position {\n pub x: i32,\n pub y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n pub id: i32,\n pub pos: Position,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(_ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn insert_entity(ctx: &ReducerContext, id: i32, x: i32, y: i32) {\n ctx.db.entity().insert(Entity { id, pos: Position { x, y } });\n}\n\n#[spacetimedb::reducer]\npub fn set_pos(ctx: &ReducerContext, id: i32, x: i32, y: i32) {\n if let Some(mut e) = ctx.db.entity().id().find(id) {\n e.pos.x = x;\n e.pos.y = y;\n ctx.db.entity().id().update(e);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-003-struct-in-table-golden", @@ -133,20 +133,21 @@ "reducers_diff": { "only_golden": [], "only_llm": [ - "add_entity()", - "init()" + "init()", + "insert_entity()", + "set_pos()" ] }, "reducers_equal": false, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T11:56:37.595509989Z", - "finished_at": "2026-01-23T11:57:06.869986441Z" + "started_at": "2026-01-23T20:45:58.319022832Z", + "finished_at": "2026-01-23T20:46:29.838664307Z" }, "t_004_insert": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -172,7 +173,7 @@ "llm_db": "basics-t-004-insert-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } @@ -181,14 +182,14 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `user`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:43035/v1/database/c2008f3693ac65719c539bc7c4992fe451a1bec99bb058273f581f5ee525695b/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `user`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c200e9fa53e3a843d1820d588423033ed683af3525a68a4dc3f8578dabd3b437/sql)\n", "phase": "sql_golden" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:56:37.595811655Z", - "finished_at": "2026-01-23T11:57:05.537662622Z" + "started_at": "2026-01-23T20:45:58.319345443Z", + "finished_at": "2026-01-23T20:46:19.313225077Z" }, "t_005_update": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -198,7 +199,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n let row = User { id, name, age, active };\n ctx.db.user().id().update(row);\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n let new_row = User { id, name, age, active };\n if ctx.db.user().id().find(id).is_some() {\n ctx.db.user().id().update(new_row);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -206,17 +207,23 @@ "work_dir_golden": "target/llm-runs/basics/t_005_update/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_005_update/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "data_parity_update_user": { "pass": true, "partial": 1.0, "notes": { + "args": [ + 1, + "Alice2", + 31, + false + ], "golden_db": "basics-t-005-update-golden", + "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "llm_db": "basics-t-005-update-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:43035", - "tables_diff": null, - "tables_equal": true + "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "query": "SELECT id, name, age, active FROM user WHERE id=1", + "reducer": "update_user", + "server": "http://127.0.0.1:34477" } }, "seed_users_row": { @@ -226,29 +233,23 @@ "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" } }, - "data_parity_update_user": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "args": [ - 1, - "Alice2", - 31, - false - ], "golden_db": "basics-t-005-update-golden", - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "llm_db": "basics-t-005-update-gpt-5-llm", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "update_user", - "server": "http://127.0.0.1:43035" + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:34477", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T11:56:59.057861578Z", - "finished_at": "2026-01-23T11:57:53.866612446Z" + "started_at": "2026-01-23T20:46:19.313268675Z", + "finished_at": "2026-01-23T20:46:50.997884926Z" }, "t_006_delete": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -258,7 +259,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, UniqueColumn};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.user().id().delete(&id);\n}", + "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.user().id().delete(&id);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-006-delete-golden", @@ -266,6 +267,13 @@ "work_dir_golden": "target/llm-runs/basics/t_006_delete/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_006_delete/rust/server/gpt-5/llm", "scorer_details": { + "seed_users_row": { + "pass": true, + "partial": 1.0, + "notes": { + "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -274,18 +282,11 @@ "llm_db": "basics-t-006-delete-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, "delete_user_count_zero": { "pass": true, "partial": 1.0, @@ -297,8 +298,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:02.840911142Z", - "finished_at": "2026-01-23T11:57:48.533997072Z" + "started_at": "2026-01-23T20:46:21.669348863Z", + "finished_at": "2026-01-23T20:46:43.139558965Z" }, "t_007_crud": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -308,7 +309,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n let users = ctx.db.user();\n\n users.insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n users.insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n\n let _ = users.id().update(User { id: 1, name: \"Alice2\".to_string(), age: 31, active: false });\n\n users.id().delete(&2);\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n let users = ctx.db.user();\n\n users.insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n users.insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n\n let updated = User { id: 1, name: \"Alice2\".to_string(), age: 31, active: false };\n let _ = users.id().update(updated);\n\n let id_to_delete = 2;\n users.id().delete(&id_to_delete);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -325,6 +326,14 @@ "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" } }, + "crud_row_id1_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `user`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c200fecea2f4fd1e842ce9bf311fc7af775ed2f1c09d17c59d34a450538b65ed/sql)\n", + "phase": "sql_golden" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -333,7 +342,7 @@ "llm_db": "basics-t-007-crud-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } @@ -346,19 +355,11 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM user" } - }, - "crud_row_id1_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `user`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:43035/v1/database/c2009e8af0d449c17ae3da6d300c97be156e1eb9f1da87b7c6b9cefbcde67475/sql)\n", - "phase": "sql_golden" - } } }, "vendor": "openai", - "started_at": "2026-01-23T11:56:37.596351347Z", - "finished_at": "2026-01-23T11:57:10.126845149Z" + "started_at": "2026-01-23T20:45:58.319932266Z", + "finished_at": "2026-01-23T20:46:41.862776693Z" }, "t_008_index_lookup": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -368,7 +369,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[reducer]\nfn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name });\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n ctx.db.result().id().delete(&u.id);\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -376,13 +377,6 @@ "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/rust/server/gpt-5/llm", "scorer_details": { - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -391,11 +385,18 @@ "llm_db": "basics-t-008-index-lookup-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } }, + "seed_user_row": { + "pass": true, + "partial": 1.0, + "notes": { + "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" + } + }, "index_lookup_projection_parity": { "pass": true, "partial": 1.0, @@ -409,13 +410,13 @@ "llm_out": "id | name ----+--------- 1 | \"Alice\"", "query": "SELECT id, name FROM result WHERE id=1", "reducer": "lookup_user_name", - "server": "http://127.0.0.1:43035" + "server": "http://127.0.0.1:34477" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:02.840644958Z", - "finished_at": "2026-01-23T11:57:40.599601481Z" + "started_at": "2026-01-23T20:46:20.508322641Z", + "finished_at": "2026-01-23T20:46:56.770679153Z" }, "t_009_init": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -425,7 +426,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -433,6 +434,15 @@ "work_dir_golden": "target/llm-runs/basics/t_009_init/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_009_init/rust/server/gpt-5/llm", "scorer_details": { + "init_seed_alice": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -441,42 +451,33 @@ "llm_db": "basics-t-009-init-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } }, - "init_total_two": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" - } - }, - "init_seed_alice": { + "init_seed_bob": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" } }, - "init_seed_bob": { + "init_total_two": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM user" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:01.112523418Z", - "finished_at": "2026-01-23T11:57:24.446822222Z" + "started_at": "2026-01-23T20:46:19.785638677Z", + "finished_at": "2026-01-23T20:46:42.338746730Z" }, "t_010_connect": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -502,15 +503,15 @@ "llm_db": "basics-t-010-connect-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T11:56:37.596220949Z", - "finished_at": "2026-01-23T11:57:02.704314900Z" + "started_at": "2026-01-23T20:45:58.319798861Z", + "finished_at": "2026-01-23T20:46:24.449136396Z" }, "t_011_helper_function": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -528,14 +529,6 @@ "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/rust/server/gpt-5/llm", "scorer_details": { - "helper_func_sum_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `result`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:43035/v1/database/c20092cdd1c98eaef99ecfeb113a11c7cee6251c76ce243f1d56372113126f59/sql)\n", - "phase": "sql_golden" - } - }, "helper_func_sum_abs": { "pass": true, "partial": 1.0, @@ -545,6 +538,14 @@ "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" } }, + "helper_func_sum_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `result`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c200a76ada80a129ab1d98b1132742dd63568f85071327226e59b33240f90ed8/sql)\n", + "phase": "sql_golden" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -553,15 +554,15 @@ "llm_db": "basics-t-011-helper-function-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T11:56:37.595665553Z", - "finished_at": "2026-01-23T11:57:02.840609277Z" + "started_at": "2026-01-23T20:45:58.319183557Z", + "finished_at": "2026-01-23T20:46:20.508270684Z" }, "t_012_spacetime_product_type": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -579,6 +580,19 @@ "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-012-spacetime-product-type-golden", + "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:34477", + "tables_diff": null, + "tables_equal": true + } + }, "product_type_row_count": { "pass": true, "partial": 1.0, @@ -603,26 +617,13 @@ "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", "query": "SELECT id, value FROM result WHERE id=1", "reducer": "set_score", - "server": "http://127.0.0.1:43035" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-012-spacetime-product-type-golden", - "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:43035", - "tables_diff": null, - "tables_equal": true + "server": "http://127.0.0.1:34477" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:40.599657560Z", - "finished_at": "2026-01-23T11:58:09.609768604Z" + "started_at": "2026-01-23T20:46:50.997932201Z", + "finished_at": "2026-01-23T20:47:18.590168533Z" }, "t_013_spacetime_sum_type": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -632,7 +633,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[reducer]\nfn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().id().delete(&id);\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -640,23 +641,6 @@ "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/gpt-5/llm", "scorer_details": { - "sum_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 10 - ], - "golden_db": "schema-t-013-spacetime-sum-type-golden", - "golden_out": "id | value ----+--------------- 1 | (Circle = 10)", - "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", - "llm_out": "id | value ----+--------------- 1 | (Circle = 10)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_circle", - "server": "http://127.0.0.1:43035" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -665,7 +649,7 @@ "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } @@ -678,11 +662,28 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" } + }, + "sum_type_row_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [ + 1, + 10 + ], + "golden_db": "schema-t-013-spacetime-sum-type-golden", + "golden_out": "id | value ----+--------------- 1 | (Circle = 10)", + "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", + "llm_out": "id | value ----+--------------- 1 | (Circle = 10)", + "query": "SELECT id, value FROM result WHERE id=1", + "reducer": "set_circle", + "server": "http://127.0.0.1:34477" + } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:48.534039773Z", - "finished_at": "2026-01-23T11:58:41.485665974Z" + "started_at": "2026-01-23T20:46:56.770721316Z", + "finished_at": "2026-01-23T20:47:22.846786446Z" }, "t_014_elementary_columns": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -692,7 +693,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000i64,\n price: 1.5f32,\n ratio: 2.25f64,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3000000000i64,\n price: 1.5f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-014-elementary-columns-golden", @@ -711,7 +712,16 @@ "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", "reducer": "seed", - "server": "http://127.0.0.1:43035" + "server": "http://127.0.0.1:34477" + } + }, + "elementary_columns_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" } }, "schema_parity": { @@ -722,34 +732,25 @@ "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" - } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:47.891667065Z", - "finished_at": "2026-01-23T11:58:20.627171437Z" + "started_at": "2026-01-23T20:46:56.319506066Z", + "finished_at": "2026-01-23T20:47:21.058802558Z" }, "t_015_product_type_columns": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", "task": "t_015_product_type_columns", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[reducer]\nfn seed(ctx: &ReducerContext) {\n let tbl = ctx.db.profile();\n for row in tbl.iter().collect::>() {\n tbl.delete(&row);\n }\n tbl.insert(Profile {\n id: 1,\n home: Address { street: \"1 Main\".to_string(), zip: 11111 },\n work: Address { street: \"2 Broad\".to_string(), zip: 22222 },\n pos: Position { x: 7, y: 9 },\n });\n}", + "passed_tests": 3, + "llm_output": "use spacetimedb::{ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".to_string(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".to_string(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -757,18 +758,46 @@ "work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling shlex v1.3.0\n Compiling either v1.15.0\n Compiling zerocopy v0.8.33\n Compiling find-msvc-tools v0.1.8\n Compiling serde v1.0.228\n Compiling anyhow v1.0.100\n Compiling nohash-hasher v0.2.0\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling zmij v1.0.16\n Compiling convert_case v0.4.0\n Compiling heck v0.4.1\n Compiling humantime v2.3.0\n Compiling arrayvec v0.7.6\n Compiling keccak v0.1.5\n Compiling bytes v1.11.0\n Compiling constant_time_eq v0.4.2\n Compiling serde_json v1.0.149\n Compiling arrayref v0.3.9\n Compiling itoa v1.0.17\n Compiling getrandom v0.2.17\n Compiling second-stack v0.3.5\n Compiling bytemuck v1.24.0\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling hex v0.4.3\n Compiling cc v1.2.54\n Compiling smallvec v1.15.1\n Compiling spacetimedb-lib v1.11.1\n Compiling memchr v2.7.6\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling blake3 v1.8.3\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_015_product_type_columns/rust/server/gpt-5/llm)\nerror[E0308]: mismatched types\n --> src/lib.rs:29:20\n |\n 29 | tbl.delete(&row);\n | ------ ^^^^ expected `Profile`, found `&Profile`\n | |\n | arguments to this method are incorrect\n |\nnote: method defined here\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/table.rs:104:8\n |\n104 | fn delete(&self, row: Self::Row) -> bool {\n | ^^^^^^\nhelp: consider removing the borrow\n |\n 29 - tbl.delete(&row);\n 29 + tbl.delete(row);\n |\n\nFor more information about this error, try `rustc --explain E0308`.\nerror: could not compile `spacetime-module` (lib) due to 1 previous error\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" + "golden_db": "schema-t-015-product-type-columns-golden", + "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:34477", + "tables_diff": null, + "tables_equal": true + } + }, + "product_type_columns_row_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [], + "golden_db": "schema-t-015-product-type-columns-golden", + "golden_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", + "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", + "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", + "query": "SELECT id, home, work, pos FROM profile WHERE id=1", + "reducer": "seed", + "server": "http://127.0.0.1:34477" + } + }, + "product_type_columns_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:51.917727311Z", - "finished_at": "2026-01-23T11:58:32.335767995Z" + "started_at": "2026-01-23T20:46:56.770961297Z", + "finished_at": "2026-01-23T20:47:32.576819834Z" }, "t_016_sum_type_columns": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -778,7 +807,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let row = Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n };\n ctx.db.drawing().insert(row);\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let a = Shape::Circle(10);\n let b = Shape::Rectangle(Rect { width: 4, height: 6 });\n ctx.db.drawing().insert(Drawing { id: 1, a, b });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -786,14 +815,6 @@ "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/gpt-5/llm", "scorer_details": { - "sum_type_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:43035/v1/database/c200fa0a7bdaa0b3fe32198acc7d7c51970587917bc93ce14e7d0e54f859979b/sql)\n", - "phase": "sql_golden" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -802,7 +823,7 @@ "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } @@ -811,14 +832,22 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:43035/v1/database/c200b5079878ebf16faffc0bc609578eb070b7c5eb623e6f0ec9ca7c1e73e040/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c2002a23ef16d310345e1d4ce4b5d48569b614371f4967976a5819022bbc2662/sql)\n", "phase": "sql" } + }, + "sum_type_columns_row_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c20040bc998fdc9ab3d5a21e438444dc0653e63e1691252800cd646cfc6722b3/sql)\n", + "phase": "sql_golden" + } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:51.917991193Z", - "finished_at": "2026-01-23T11:58:22.938115353Z" + "started_at": "2026-01-23T20:47:00.920013549Z", + "finished_at": "2026-01-23T20:47:31.508612717Z" }, "t_017_scheduled_columns": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -828,7 +857,7 @@ "model_name": "GPT-5", "total_tests": 2, "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(scheduled)]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n // Scheduled tick; no-op for this task.\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n }\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -840,14 +869,14 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling either v1.15.0\n Compiling serde v1.0.228\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.33\n Compiling find-msvc-tools v0.1.8\n Compiling anyhow v1.0.100\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling nohash-hasher v0.2.0\n Compiling arrayvec v0.7.6\n Compiling zmij v1.0.16\n Compiling humantime v2.3.0\n Compiling convert_case v0.4.0\n Compiling keccak v0.1.5\n Compiling heck v0.4.1\n Compiling bytes v1.11.0\n Compiling serde_json v1.0.149\n Compiling constant_time_eq v0.4.2\n Compiling second-stack v0.3.5\n Compiling spacetimedb-lib v1.11.1\n Compiling getrandom v0.2.17\n Compiling arrayref v0.3.9\n Compiling smallvec v1.15.1\n Compiling cc v1.2.54\n Compiling itoa v1.0.17\n Compiling bytemuck v1.24.0\n Compiling hex v0.4.3\n Compiling itertools v0.12.1\n Compiling rand_core v0.6.4\n Compiling memchr v2.7.6\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling blake3 v1.8.3\n Compiling decorum v0.3.1\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror: expected one of: `init`, `client_connected`, `client_disconnected`, `update`, `name`\n --> src/lib.rs:12:11\n |\n12 | #[reducer(scheduled)]\n | ^^^^^^^^^\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:13:42\n |\n13 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:20:36\n |\n20 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:19:15\n |\n19 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:20:16\n |\n20 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:22:39\n |\n22 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0412`.\nerror: could not compile `spacetime-module` (lib) due to 7 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling zerocopy v0.8.33\n Compiling shlex v1.3.0\n Compiling find-msvc-tools v0.1.8\n Compiling bitflags v2.10.0\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling thiserror v1.0.69\n Compiling bytes v1.11.0\n Compiling convert_case v0.4.0\n Compiling keccak v0.1.5\n Compiling arrayvec v0.7.6\n Compiling heck v0.4.1\n Compiling zmij v1.0.16\n Compiling humantime v2.3.0\n Compiling second-stack v0.3.5\n Compiling bytemuck v1.24.0\n Compiling itoa v1.0.17\n Compiling arrayref v0.3.9\n Compiling getrandom v0.2.17\n Compiling constant_time_eq v0.4.2\n Compiling hex v0.4.3\n Compiling itertools v0.12.1\n Compiling spacetimedb-lib v1.11.1\n Compiling serde_json v1.0.149\n Compiling smallvec v1.15.1\n Compiling rand_core v0.6.4\n Compiling memchr v2.7.6\n Compiling generic-array v0.14.7\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling num-traits v0.2.19\n Compiling cc v1.2.54\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling blake3 v1.8.3\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:15:36\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:23:42\n |\n23 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {}\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:15\n |\n14 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:15:16\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:17:39\n |\n17 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:23:8\n |\n 22 | #[reducer]\n | ---------- required by a bound introduced by this call\n 23 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {}\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `register_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:746:81\n |\n746 | pub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl Reducer<'a, A>) {\n | ^^^^^^^^^^^^^^ required by this bound in `register_reducer`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:23:8\n |\n22 | #[reducer]\n | ---------- required by a bound introduced by this call\n23 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {}\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `invoke_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:45:19\n |\n44 | pub fn invoke_reducer<'a, A: Args<'a>>(\n | -------------- required by a bound in this function\n45 | reducer: impl Reducer<'a, A>,\n | ^^^^^^^^^^^^^^ required by this bound in `invoke_reducer`\n\nSome errors have detailed explanations: E0277, E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0277`.\nerror: could not compile `spacetime-module` (lib) due to 8 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:31.320145539Z", - "finished_at": "2026-01-23T11:58:26.729342562Z" + "started_at": "2026-01-23T20:46:43.139606842Z", + "finished_at": "2026-01-23T20:47:30.076456163Z" }, "t_018_constraints": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -857,7 +886,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[reducer]\nfn seed(ctx: &ReducerContext) {\n let tbl = ctx.db.account();\n\n let ids: Vec = tbl.iter().map(|r| r.id).collect();\n for id in ids {\n tbl.id().delete(&id);\n }\n\n tbl.insert(Account { id: 1, email: \"a@example.com\".to_string(), name: \"Alice\".to_string() });\n tbl.insert(Account { id: 2, email: \"b@example.com\".to_string(), name: \"Bob\".to_string() });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let tbl = ctx.db.account();\n\n for row in tbl.iter() {\n tbl.id().delete(&row.id);\n }\n\n tbl.insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n tbl.insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-018-constraints-golden", @@ -865,6 +894,19 @@ "work_dir_golden": "target/llm-runs/schema/t_018_constraints/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_018_constraints/rust/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-018-constraints-golden", + "llm_db": "schema-t-018-constraints-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:34477", + "tables_diff": null, + "tables_equal": true + } + }, "constraints_seed_two_rows": { "pass": true, "partial": 1.0, @@ -885,26 +927,13 @@ "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", "query": "SELECT id, email, name FROM account WHERE id=1", "reducer": "seed", - "server": "http://127.0.0.1:43035" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-018-constraints-golden", - "llm_db": "schema-t-018-constraints-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:43035", - "tables_diff": null, - "tables_equal": true + "server": "http://127.0.0.1:34477" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:10.126874202Z", - "finished_at": "2026-01-23T11:57:47.891616379Z" + "started_at": "2026-01-23T20:46:41.862819773Z", + "finished_at": "2026-01-23T20:47:07.104266828Z" }, "t_019_many_to_many": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -914,7 +943,7 @@ "model_name": "GPT-5", "total_tests": 5, "passed_tests": 5, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.membership().iter() {\n ctx.db.membership().id().delete(&row.id);\n }\n for row in ctx.db.user().iter() {\n ctx.db.user().user_id().delete(&row.user_id);\n }\n for row in ctx.db.group().iter() {\n ctx.db.group().group_id().delete(&row.group_id);\n }\n\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[reducer]\nfn seed(ctx: &ReducerContext) {\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-019-many-to-many-golden", @@ -940,15 +969,6 @@ "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" } }, - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } - }, "m2m_has_1_20": { "pass": true, "partial": 1.0, @@ -966,15 +986,24 @@ "llm_db": "schema-t-019-many-to-many-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } + }, + "memberships_three_rows": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM membership" + } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:24.446860971Z", - "finished_at": "2026-01-23T11:58:08.134821703Z" + "started_at": "2026-01-23T20:46:42.338778389Z", + "finished_at": "2026-01-23T20:47:21.898328928Z" }, "t_020_ecs": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -984,7 +1013,7 @@ "model_name": "GPT-5", "total_tests": 5, "passed_tests": 1, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n // Entities\n let entities = ctx.db.entity();\n if entities.id().find(1).is_none() {\n entities.insert(Entity { id: 1 });\n }\n if entities.id().find(2).is_none() {\n entities.insert(Entity { id: 2 });\n }\n\n // Positions\n let positions = ctx.db.position();\n if let Some(mut p) = positions.entity_id().find(1) {\n p.x = 0;\n p.y = 0;\n let _ = positions.entity_id().update(p);\n } else {\n positions.insert(Position { entity_id: 1, x: 0, y: 0 });\n }\n if let Some(mut p) = positions.entity_id().find(2) {\n p.x = 10;\n p.y = 0;\n let _ = positions.entity_id().update(p);\n } else {\n positions.insert(Position { entity_id: 2, x: 10, y: 0 });\n }\n\n // Velocities\n let velocities = ctx.db.velocity();\n if let Some(mut v) = velocities.entity_id().find(1) {\n v.vx = 1;\n v.vy = 0;\n let _ = velocities.entity_id().update(v);\n } else {\n velocities.insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n }\n if let Some(mut v) = velocities.entity_id().find(2) {\n v.vx = -2;\n v.vy = 3;\n let _ = velocities.entity_id().update(v);\n } else {\n velocities.insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n }\n}\n\n#[reducer]\npub fn step(ctx: &ReducerContext) {\n let positions = ctx.db.position();\n let velocities = ctx.db.velocity();\n let next_positions = ctx.db.next_position();\n\n for pos in positions.iter() {\n if let Some(vel) = velocities.entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n\n if let Some(mut np) = next_positions.entity_id().find(pos.entity_id) {\n np.x = nx;\n np.y = ny;\n let _ = next_positions.entity_id().update(np);\n } else {\n next_positions.insert(NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n });\n }\n }\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, UniqueColumn};\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let _ = ctx.db.entity().try_insert(Entity { id: 1 });\n let _ = ctx.db.entity().try_insert(Entity { id: 2 });\n\n let _ = ctx.db.position().try_insert(Position { entity_id: 1, x: 0, y: 0 });\n let _ = ctx.db.position().try_insert(Position { entity_id: 2, x: 10, y: 0 });\n\n let _ = ctx.db.velocity().try_insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n let _ = ctx.db.velocity().try_insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n}\n\n#[reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n let next = NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n };\n if ctx.db.next_position().entity_id().find(pos.entity_id).is_some() {\n let _ = ctx.db.next_position().entity_id().update(next);\n } else {\n let _ = ctx.db.next_position().insert(next);\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -992,19 +1021,11 @@ "work_dir_golden": "target/llm-runs/schema/t_020_ecs/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_020_ecs/rust/server/gpt-5/llm", "scorer_details": { - "ecs_next_pos_entity1": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:43035/v1/database/c200bfacda1977a80778696e02687f4296607c0e5dc2757b9dc0bd6ed69ac556/sql)\n", - "phase": "sql" - } - }, - "ecs_next_pos_entity2": { + "ecs_seed_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:43035/v1/database/c200bfacda1977a80778696e02687f4296607c0e5dc2757b9dc0bd6ed69ac556/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c2002015adb1317fe3abcb13d934628242ad331957f4caa79e06dee69a06a590/sql)\n", "phase": "sql" } }, @@ -1016,7 +1037,7 @@ "llm_db": "schema-t-020-ecs-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } @@ -1025,22 +1046,30 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:43035/v1/database/c200bfacda1977a80778696e02687f4296607c0e5dc2757b9dc0bd6ed69ac556/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c2002015adb1317fe3abcb13d934628242ad331957f4caa79e06dee69a06a590/sql)\n", "phase": "sql" } }, - "ecs_seed_positions_count": { + "ecs_next_pos_entity1": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c2002015adb1317fe3abcb13d934628242ad331957f4caa79e06dee69a06a590/sql)\n", + "phase": "sql" + } + }, + "ecs_next_pos_entity2": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:43035/v1/database/c200bfacda1977a80778696e02687f4296607c0e5dc2757b9dc0bd6ed69ac556/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c2002015adb1317fe3abcb13d934628242ad331957f4caa79e06dee69a06a590/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:05.537695688Z", - "finished_at": "2026-01-23T11:57:51.917639786Z" + "started_at": "2026-01-23T20:46:24.449168444Z", + "finished_at": "2026-01-23T20:47:00.919947661Z" }, "t_021_multi_column_index": { "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", @@ -1050,7 +1079,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let t = ctx.db.log();\n t.insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n t.insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n t.insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -1066,7 +1095,7 @@ "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:43035", + "server": "http://127.0.0.1:34477", "tables_diff": null, "tables_equal": true } @@ -1100,8 +1129,8 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:57:06.870008743Z", - "finished_at": "2026-01-23T11:57:31.320114801Z" + "started_at": "2026-01-23T20:46:29.838705690Z", + "finished_at": "2026-01-23T20:46:56.594909460Z" } } } @@ -1109,21 +1138,21 @@ }, { "mode": "docs", - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "models": [ { "name": "GPT-5", "route_api_model": "gpt-5", "tasks": { "t_000_empty_reducers": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_000_empty_reducers", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, count: i32) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, count: i32, name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, active: bool, ratio: f32, label: String) {}", + "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, _count: i32) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, _count: i32, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, _active: bool, _ratio: f32, _label: String) {}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-000-empty-reducers-golden", @@ -1141,11 +1170,11 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:58:41.716223450Z", - "finished_at": "2026-01-23T11:59:49.748708946Z" + "started_at": "2026-01-23T20:47:32.846152158Z", + "finished_at": "2026-01-23T20:48:46.489399789Z" }, "t_001_basic_tables": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_001_basic_tables", "lang": "rust", "golden_published": true, @@ -1170,18 +1199,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:58:41.704113020Z", - "finished_at": "2026-01-23T11:59:42.429041793Z" + "started_at": "2026-01-23T20:47:32.833609179Z", + "finished_at": "2026-01-23T20:48:40.565099319Z" }, "t_002_scheduled_table": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_002_scheduled_table", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use std::time::Duration;\nuse spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) -> Result<(), String> {\n Ok(())\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n let desired = ScheduleAt::Interval(Duration::from_micros(50_000).into());\n let count = ctx.db.tick_timer().count();\n if count == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: desired,\n });\n } else {\n let mut iter = ctx.db.tick_timer().iter();\n if let Some(mut first) = iter.next() {\n first.scheduled_at = desired;\n ctx.db.tick_timer().scheduled_id().update(first);\n }\n for extra in iter {\n ctx.db.tick_timer().scheduled_id().delete(&extra.scheduled_id);\n }\n }\n Ok(())\n}", + "llm_output": "use std::time::Duration;\nuse spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -1199,18 +1228,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:58:41.719410738Z", - "finished_at": "2026-01-23T11:59:57.789054932Z" + "started_at": "2026-01-23T20:47:32.849575285Z", + "finished_at": "2026-01-23T20:48:55.497529150Z" }, "t_003_struct_in_table": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_003_struct_in_table", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::SpacetimeType;\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n pub x: i32,\n pub y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n pub id: i32,\n pub pos: Position,\n}", + "llm_output": "use spacetimedb::SpacetimeType;\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-003-struct-in-table-golden", @@ -1228,11 +1257,11 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:58:41.706682369Z", - "finished_at": "2026-01-23T11:59:26.096561811Z" + "started_at": "2026-01-23T20:47:32.836252652Z", + "finished_at": "2026-01-23T20:48:05.833400632Z" }, "t_004_insert": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_004_insert", "lang": "rust", "golden_published": true, @@ -1265,18 +1294,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:58:41.712966773Z", - "finished_at": "2026-01-23T11:59:51.858461717Z" + "started_at": "2026-01-23T20:47:32.842787873Z", + "finished_at": "2026-01-23T20:48:29.038416446Z" }, "t_005_update": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_005_update", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n if ctx.db.user().id().find(id).is_some() {\n ctx.db.user().id().update(User { id, name, age, active });\n }\n}", + "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().id().update(User { id, name, age, active });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -1284,12 +1313,12 @@ "work_dir_golden": "target/llm-runs/basics/t_005_update/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_005_update/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "data_parity_update_user": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", - "phase": "describe_golden" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", + "phase": "call_reducer_golden" } }, "seed_users_row": { @@ -1301,21 +1330,21 @@ "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" } }, - "data_parity_update_user": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", - "phase": "call_reducer_golden" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", + "phase": "describe_golden" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:59:26.096589408Z", - "finished_at": "2026-01-23T11:59:53.555166464Z" + "started_at": "2026-01-23T20:48:05.833493117Z", + "finished_at": "2026-01-23T20:48:31.203040278Z" }, "t_006_delete": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_006_delete", "lang": "rust", "golden_published": true, @@ -1330,6 +1359,14 @@ "work_dir_golden": "target/llm-runs/basics/t_006_delete/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_006_delete/rust/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-006-delete-golden`.\n", + "phase": "describe_golden" + } + }, "seed_users_row": { "pass": false, "partial": 0.0, @@ -1347,22 +1384,14 @@ "expected": 0, "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-006-delete-golden`.\n", - "phase": "describe_golden" - } } }, "vendor": "openai", - "started_at": "2026-01-23T11:59:43.249955612Z", - "finished_at": "2026-01-23T12:00:06.215097718Z" + "started_at": "2026-01-23T20:48:31.203078568Z", + "finished_at": "2026-01-23T20:48:50.906207073Z" }, "t_007_crud": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_007_crud", "lang": "rust", "golden_published": true, @@ -1377,6 +1406,14 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", + "phase": "describe_golden" + } + }, "crud_row_id2_deleted": { "pass": true, "partial": 1.0, @@ -1386,12 +1423,13 @@ "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" } }, - "schema_parity": { + "crud_total_count_one": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", - "phase": "describe_golden" + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user" } }, "crud_row_id1_parity": { @@ -1401,30 +1439,21 @@ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", "phase": "call_reducer_golden" } - }, - "crud_total_count_one": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" - } } }, "vendor": "openai", - "started_at": "2026-01-23T11:58:41.725929053Z", - "finished_at": "2026-01-23T11:59:43.249835351Z" + "started_at": "2026-01-23T20:47:32.856090158Z", + "finished_at": "2026-01-23T20:48:19.458976831Z" }, "t_008_index_lookup": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_008_index_lookup", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(user) = ctx.db.user().id().find(id) {\n ctx.db.result().insert(ResultRow { id: user.id, name: user.name });\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name.clone() });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -1459,18 +1488,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:59:42.429072085Z", - "finished_at": "2026-01-23T12:00:06.978561278Z" + "started_at": "2026-01-23T20:48:29.038498151Z", + "finished_at": "2026-01-23T20:48:56.042485955Z" }, "t_009_init": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_009_init", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".into(),\n age: 30,\n active: true,\n });\n ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".into(),\n age: 22,\n active: false,\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".into(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".into(), age: 22, active: false });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -1478,13 +1507,21 @@ "work_dir_golden": "target/llm-runs/basics/t_009_init/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_009_init/rust/server/gpt-5/llm", "scorer_details": { - "init_total_two": { + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-009-init-golden`.\n", + "phase": "describe_golden" + } + }, + "init_seed_bob": { "pass": true, "partial": 1.0, "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" } }, "init_seed_alice": { @@ -1496,37 +1533,29 @@ "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" } }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-009-init-golden`.\n", - "phase": "describe_golden" - } - }, - "init_seed_bob": { + "init_total_two": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM user" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:59:40.767658962Z", - "finished_at": "2026-01-23T12:00:00.250974348Z" + "started_at": "2026-01-23T20:48:19.459032083Z", + "finished_at": "2026-01-23T20:48:44.457914124Z" }, "t_010_connect": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_010_connect", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = event)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event { id: 0, kind: \"connected\".into() });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event { id: 0, kind: \"disconnected\".into() });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = event)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event {\n id: 0,\n kind: \"connected\".to_string(),\n });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event {\n id: 0,\n kind: \"disconnected\".to_string(),\n });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-010-connect-golden", @@ -1544,11 +1573,11 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:58:41.722667055Z", - "finished_at": "2026-01-23T11:59:44.003413383Z" + "started_at": "2026-01-23T20:47:32.852844068Z", + "finished_at": "2026-01-23T20:48:51.666864219Z" }, "t_011_helper_function": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_011_helper_function", "lang": "rust", "golden_published": true, @@ -1563,15 +1592,6 @@ "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/rust/server/gpt-5/llm", "scorer_details": { - "helper_func_sum_abs": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" - } - }, "schema_parity": { "pass": false, "partial": 0.0, @@ -1587,21 +1607,30 @@ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-011-helper-function-golden`.\n", "phase": "call_reducer_golden" } + }, + "helper_func_sum_abs": { + "pass": false, + "partial": 0.0, + "notes": { + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" + } } }, "vendor": "openai", - "started_at": "2026-01-23T11:58:41.709720362Z", - "finished_at": "2026-01-23T11:59:40.767569416Z" + "started_at": "2026-01-23T20:47:32.839497474Z", + "finished_at": "2026-01-23T20:48:51.151329754Z" }, "t_012_spacetime_product_type": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_012_spacetime_product_type", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[derive(spacetimedb::SpacetimeType, Clone, Debug)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n let value = Score { left, right };\n ctx.db.result().insert(ResultRow { id, value });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -1609,20 +1638,20 @@ "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "product_type_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-012-spacetime-product-type-golden`.\n", - "phase": "describe_golden" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-012-spacetime-product-type-golden`.\n", + "phase": "call_reducer_golden" } }, - "product_type_row_parity": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-012-spacetime-product-type-golden`.\n", - "phase": "call_reducer_golden" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-012-spacetime-product-type-golden`.\n", + "phase": "describe_golden" } }, "product_type_row_count": { @@ -1636,18 +1665,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T12:00:00.251008970Z", - "finished_at": "2026-01-23T12:00:21.554582472Z" + "started_at": "2026-01-23T20:48:51.666947087Z", + "finished_at": "2026-01-23T20:49:14.419221949Z" }, "t_013_spacetime_sum_type": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_013_spacetime_sum_type", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -1655,6 +1684,14 @@ "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-013-spacetime-sum-type-golden`.\n", + "phase": "describe_golden" + } + }, "sum_type_row_count": { "pass": false, "partial": 0.0, @@ -1671,29 +1708,21 @@ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-013-spacetime-sum-type-golden`.\n", "phase": "call_reducer_golden" } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-013-spacetime-sum-type-golden`.\n", - "phase": "describe_golden" - } } }, "vendor": "openai", - "started_at": "2026-01-23T12:00:06.978584823Z", - "finished_at": "2026-01-23T12:00:33.966822730Z" + "started_at": "2026-01-23T20:48:56.042516286Z", + "finished_at": "2026-01-23T20:49:19.391193690Z" }, "t_014_elementary_columns": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_014_elementary_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5f32,\n ratio: 2.25f64,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-014-elementary-columns-golden", @@ -1701,14 +1730,6 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/rust/server/gpt-5/llm", "scorer_details": { - "elementary_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", - "phase": "call_reducer_golden" - } - }, "elementary_columns_row_count": { "pass": false, "partial": 0.0, @@ -1725,21 +1746,29 @@ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", "phase": "describe_golden" } + }, + "elementary_columns_row_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", + "phase": "call_reducer_golden" + } } }, "vendor": "openai", - "started_at": "2026-01-23T12:00:06.215125322Z", - "finished_at": "2026-01-23T12:00:33.902324216Z" + "started_at": "2026-01-23T20:48:55.497603714Z", + "finished_at": "2026-01-23T20:49:21.590560437Z" }, "t_015_product_type_columns": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_015_product_type_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address { street: \"1 Main\".into(), zip: 11111 },\n work: Address { street: \"2 Broad\".into(), zip: 22222 },\n pos: Position { x: 7, y: 9 },\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.profile().iter() {\n ctx.db.profile().id().delete(&row.id);\n }\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".to_string(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".to_string(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -1774,18 +1803,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T12:00:18.383198165Z", - "finished_at": "2026-01-23T12:01:34.378780677Z" + "started_at": "2026-01-23T20:49:09.915973191Z", + "finished_at": "2026-01-23T20:49:35.664676341Z" }, "t_016_sum_type_columns": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_016_sum_type_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.drawing().iter() {\n ctx.db.drawing().id().delete(row.id);\n }\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -1793,12 +1822,12 @@ "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/gpt-5/llm", "scorer_details": { - "sum_type_columns_row_parity": { + "sum_type_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", - "phase": "call_reducer_golden" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38097/v1/database/c200de67ac503f1e38be9c09680d3df61e49eb3a7d66ef3c6c8c371fb66d4edf/sql)\n", + "phase": "sql" } }, "schema_parity": { @@ -1809,28 +1838,28 @@ "phase": "describe_golden" } }, - "sum_type_columns_row_count": { + "sum_type_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40975/v1/database/c200b73d487f28a58ba0c1daee3f8e73c716071a8ecb6ca5512534eaaf3f94f4/sql)\n", - "phase": "sql" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", + "phase": "call_reducer_golden" } } }, "vendor": "openai", - "started_at": "2026-01-23T12:00:21.554609433Z", - "finished_at": "2026-01-23T12:00:50.848757403Z" + "started_at": "2026-01-23T20:49:10.455837761Z", + "finished_at": "2026-01-23T20:49:49.430676143Z" }, "t_017_scheduled_columns": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_017_scheduled_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 2, "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, ScheduleAt, Table};\nuse std::time::Duration;\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.tick_timer().try_insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n })?;\n Ok(())\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) -> Result<(), String> {\n Ok(())\n}", + "llm_output": "use std::time::Duration;\nuse spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -1857,18 +1886,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:59:57.789164943Z", - "finished_at": "2026-01-23T12:00:37.720475314Z" + "started_at": "2026-01-23T20:48:51.151415261Z", + "finished_at": "2026-01-23T20:49:22.096623158Z" }, "t_018_constraints": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_018_constraints", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let ids: Vec = ctx.db.account().iter().map(|a| a.id).collect();\n for id in ids {\n ctx.db.account().id().delete(id);\n }\n\n ctx.db.account().insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n ctx.db.account().insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.account().insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n ctx.db.account().insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-018-constraints-golden", @@ -1903,83 +1932,47 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T11:59:51.858540280Z", - "finished_at": "2026-01-23T12:00:18.383168342Z" + "started_at": "2026-01-23T20:48:46.489485052Z", + "finished_at": "2026-01-23T20:49:14.344881271Z" }, "t_019_many_to_many": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_019_many_to_many", "lang": "rust", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", - "total_tests": 5, - "passed_tests": 4, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[spacetimedb::table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[spacetimedb::table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Ensure exactly these rows exist by clearing existing state.\n // Delete memberships first to avoid FK-like logical dependencies.\n let membership_ids: Vec = ctx.db.membership().iter().map(|m| m.id).collect();\n for id in membership_ids {\n ctx.db.membership().id().delete(&id);\n }\n\n let user_ids: Vec = ctx.db.user().iter().map(|u| u.user_id).collect();\n for id in user_ids {\n ctx.db.user().user_id().delete(&id);\n }\n\n let group_ids: Vec = ctx.db.group().iter().map(|g| g.group_id).collect();\n for id in group_ids {\n ctx.db.group().group_id().delete(&id);\n }\n\n // Insert users\n ctx.db.user().insert(User {\n user_id: 1,\n name: \"Alice\".to_string(),\n });\n ctx.db.user().insert(User {\n user_id: 2,\n name: \"Bob\".to_string(),\n });\n\n // Insert groups\n ctx.db.group().insert(Group {\n group_id: 10,\n title: \"Admin\".to_string(),\n });\n ctx.db.group().insert(Group {\n group_id: 20,\n title: \"Dev\".to_string(),\n });\n\n // Insert memberships\n ctx.db.membership().insert(Membership {\n id: 1,\n user_id: 1,\n group_id: 10,\n });\n ctx.db.membership().insert(Membership {\n id: 2,\n user_id: 1,\n group_id: 20,\n });\n ctx.db.membership().insert(Membership {\n id: 3,\n user_id: 2,\n group_id: 20,\n });\n}", + "total_tests": 1, + "passed_tests": 0, + "llm_output": null, "category": "schema", "route_api_model": "gpt-5", - "golden_db": "schema-t-019-many-to-many-golden", - "llm_db": "schema-t-019-many-to-many-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/rust/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/rust/server/gpt-5/llm", + "golden_db": null, + "llm_db": null, + "work_dir_golden": null, + "work_dir_llm": null, "scorer_details": { - "memberships_three_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" - } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" - } - }, - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" - } - }, - "m2m_has_2_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" - } - }, - "schema_parity": { + "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-019-many-to-many-golden`.\n", - "phase": "describe_golden" + "error": "POST https://api.openai.com/v1/responses -> 502 Bad Gateway: \r\n502 Bad Gateway\r\n\r\n

502 Bad Gateway

\r\n
cloudflare
\r\n\r\n\r\n", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:59:53.555195009Z", - "finished_at": "2026-01-23T12:00:37.548228808Z" + "started_at": "2026-01-23T20:49:49.431161113Z", + "finished_at": "2026-01-23T20:49:49.431161113Z" }, "t_020_ecs": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_020_ecs", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 5, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Entity 1\n if ctx.db.entity().id().find(1).is_none() {\n ctx.db.entity().insert(Entity { id: 1 });\n }\n if ctx.db.position().entity_id().find(1).is_some() {\n ctx.db.position().entity_id().update(Position { entity_id: 1, x: 0, y: 0 });\n } else {\n ctx.db.position().insert(Position { entity_id: 1, x: 0, y: 0 });\n }\n if ctx.db.velocity().entity_id().find(1).is_some() {\n ctx.db.velocity().entity_id().update(Velocity { entity_id: 1, vx: 1, vy: 0 });\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n }\n\n // Entity 2\n if ctx.db.entity().id().find(2).is_none() {\n ctx.db.entity().insert(Entity { id: 2 });\n }\n if ctx.db.position().entity_id().find(2).is_some() {\n ctx.db.position().entity_id().update(Position { entity_id: 2, x: 10, y: 0 });\n } else {\n ctx.db.position().insert(Position { entity_id: 2, x: 10, y: 0 });\n }\n if ctx.db.velocity().entity_id().find(2).is_some() {\n ctx.db.velocity().entity_id().update(Velocity { entity_id: 2, vx: -2, vy: 3 });\n } else {\n ctx.db.velocity().insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n }\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for p in ctx.db.position().iter() {\n if let Some(v) = ctx.db.velocity().entity_id().find(p.entity_id) {\n let row = NextPosition {\n entity_id: p.entity_id,\n x: p.x + v.vx,\n y: p.y + v.vy,\n };\n if ctx.db.next_position().entity_id().find(p.entity_id).is_some() {\n ctx.db.next_position().entity_id().update(row);\n } else {\n ctx.db.next_position().insert(row);\n }\n }\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let _ = ctx.db.entity().try_insert(Entity { id: 1 });\n let _ = ctx.db.entity().try_insert(Entity { id: 2 });\n\n let _ = ctx.db.position().try_insert(Position { entity_id: 1, x: 0, y: 0 });\n let _ = ctx.db.velocity().try_insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n\n let _ = ctx.db.position().try_insert(Position { entity_id: 2, x: 10, y: 0 });\n let _ = ctx.db.velocity().try_insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n if let Some(mut existing) = ctx.db.next_position().entity_id().find(pos.entity_id) {\n existing.x = nx;\n existing.y = ny;\n ctx.db.next_position().entity_id().update(existing);\n } else {\n ctx.db.next_position().insert(NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n });\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -1995,52 +1988,52 @@ "phase": "describe_golden" } }, - "ecs_next_pos_entity1": { + "ecs_next_pos_entity2": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40975/v1/database/c2001baf932156f49884b12b75839c4ef389d8ab4c55ac377157c4c2d21b7f10/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38097/v1/database/c200af80055dc33059e50f551016932cbf26d8f0f12ac2d6c5d1b5939a8d9aa1/sql)\n", "phase": "sql" } }, - "ecs_step_next_positions_count": { + "ecs_next_pos_entity1": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40975/v1/database/c2001baf932156f49884b12b75839c4ef389d8ab4c55ac377157c4c2d21b7f10/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38097/v1/database/c200af80055dc33059e50f551016932cbf26d8f0f12ac2d6c5d1b5939a8d9aa1/sql)\n", "phase": "sql" } }, - "ecs_next_pos_entity2": { + "ecs_seed_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40975/v1/database/c2001baf932156f49884b12b75839c4ef389d8ab4c55ac377157c4c2d21b7f10/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38097/v1/database/c200af80055dc33059e50f551016932cbf26d8f0f12ac2d6c5d1b5939a8d9aa1/sql)\n", "phase": "sql" } }, - "ecs_seed_positions_count": { + "ecs_step_next_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40975/v1/database/c2001baf932156f49884b12b75839c4ef389d8ab4c55ac377157c4c2d21b7f10/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38097/v1/database/c200af80055dc33059e50f551016932cbf26d8f0f12ac2d6c5d1b5939a8d9aa1/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:59:44.003507320Z", - "finished_at": "2026-01-23T12:00:45.914588769Z" + "started_at": "2026-01-23T20:48:40.565127071Z", + "finished_at": "2026-01-23T20:49:16.253635323Z" }, "t_021_multi_column_index": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "task": "t_021_multi_column_index", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.log().iter() {\n ctx.db.log().id().delete(row.id);\n }\n ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".into() });\n ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".into() });\n ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".into() });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -2048,15 +2041,6 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/rust/server/gpt-5/llm", "scorer_details": { - "mcindex_lookup_u7_d2": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" - } - }, "mcindex_seed_count": { "pass": true, "partial": 1.0, @@ -2066,6 +2050,14 @@ "sql": "SELECT COUNT(*) AS n FROM log" } }, + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-021-multi-column-index-golden`.\n", + "phase": "describe_golden" + } + }, "mcindex_lookup_u7_d1": { "pass": true, "partial": 1.0, @@ -2075,18 +2067,19 @@ "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" } }, - "schema_parity": { - "pass": false, - "partial": 0.0, + "mcindex_lookup_u7_d2": { + "pass": true, + "partial": 1.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-021-multi-column-index-golden`.\n", - "phase": "describe_golden" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" } } }, "vendor": "openai", - "started_at": "2026-01-23T11:59:49.748795415Z", - "finished_at": "2026-01-23T12:00:23.844832103Z" + "started_at": "2026-01-23T20:48:44.457949164Z", + "finished_at": "2026-01-23T20:49:10.455801685Z" } } } @@ -2277,14 +2270,14 @@ "modes": [ { "mode": "docs", - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "models": [ { "name": "GPT-5", "route_api_model": "gpt-5", "tasks": { "t_000_empty_reducers": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_000_empty_reducers", "lang": "csharp", "golden_published": true, @@ -2307,25 +2300,25 @@ "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:46871", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T12:04:18.656745307Z", - "finished_at": "2026-01-23T12:04:50.054718472Z" + "started_at": "2026-01-23T20:51:58.516007671Z", + "finished_at": "2026-01-23T20:52:19.852644961Z" }, "t_001_basic_tables": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_001_basic_tables", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Product\", Public = true)]\n public partial struct Product\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table(Name = \"Note\", Public = true)]\n public partial struct Note\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Product\")]\n public partial struct Product\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table(Name = \"Note\")]\n public partial struct Note\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-001-basic-tables-golden", @@ -2341,25 +2334,25 @@ "llm_db": "basics-t-001-basic-tables-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:46871", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T12:03:14.882300132Z", - "finished_at": "2026-01-23T12:04:36.458730908Z" + "started_at": "2026-01-23T20:51:08.718544439Z", + "finished_at": "2026-01-23T20:51:58.515971496Z" }, "t_002_scheduled_table": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_002_scheduled_table", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer _timer)\n {\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n if (ctx.Db.TickTimer.Count == 0)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n }\n}", + "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer tickTimer)\n {\n // Scheduled reducer invoked by TickTimer; no-op by specification.\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n if (ctx.Db.TickTimer.Count == 0)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -2375,18 +2368,18 @@ "llm_db": "basics-t-002-scheduled-table-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:46871", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T12:04:24.844091061Z", - "finished_at": "2026-01-23T12:05:24.876865213Z" + "started_at": "2026-01-23T20:52:02.545010135Z", + "finished_at": "2026-01-23T20:52:47.129183717Z" }, "t_003_struct_in_table": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_003_struct_in_table", "lang": "csharp", "golden_published": true, @@ -2409,25 +2402,25 @@ "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:46871", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T12:03:14.885907238Z", - "finished_at": "2026-01-23T12:04:30.187542427Z" + "started_at": "2026-01-23T20:51:08.721057407Z", + "finished_at": "2026-01-23T20:52:05.996428958Z" }, "t_004_insert": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_004_insert", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 2, "passed_tests": 2, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.User.Insert(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void InsertUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.User.Insert(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-004-insert-golden", @@ -2443,7 +2436,7 @@ "llm_db": "basics-t-004-insert-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:46871", "tables_diff": null, "tables_equal": true } @@ -2464,23 +2457,23 @@ "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "InsertUser", - "server": "http://127.0.0.1:35813" + "server": "http://127.0.0.1:46871" } } }, "vendor": "openai", - "started_at": "2026-01-23T12:03:14.897716646Z", - "finished_at": "2026-01-23T12:04:18.656638441Z" + "started_at": "2026-01-23T20:51:08.727492764Z", + "finished_at": "2026-01-23T20:52:08.554796558Z" }, "t_005_update": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_005_update", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n var existing = ctx.Db.User.Id.Find(id);\n if (existing == null)\n {\n throw new System.Exception(\"User not found\");\n }\n\n var updated = new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n };\n ctx.Db.User.Id.Update(updated);\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n var updated = new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n };\n\n if (ctx.Db.User.Id.Find(id) != null)\n {\n ctx.Db.User.Id.Update(updated);\n }\n else\n {\n ctx.Db.User.Insert(updated);\n }\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -2488,6 +2481,19 @@ "work_dir_golden": "target/llm-runs/basics/t_005_update/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_005_update/csharp/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-005-update-golden", + "llm_db": "basics-t-005-update-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:46871", + "tables_diff": null, + "tables_equal": true + } + }, "data_parity_update_user": { "pass": true, "partial": 1.0, @@ -2504,7 +2510,7 @@ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "UpdateUser", - "server": "http://127.0.0.1:35813" + "server": "http://127.0.0.1:46871" } }, "seed_users_row": { @@ -2513,34 +2519,21 @@ "notes": { "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-005-update-golden", - "llm_db": "basics-t-005-update-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:35813", - "tables_diff": null, - "tables_equal": true - } } }, "vendor": "openai", - "started_at": "2026-01-23T12:04:50.054760665Z", - "finished_at": "2026-01-23T12:05:47.466386003Z" + "started_at": "2026-01-23T20:52:19.852673232Z", + "finished_at": "2026-01-23T20:52:53.389808270Z" }, "t_006_delete": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_006_delete", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.User.Id.Delete(id);\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.User.Id.Delete(id);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-006-delete-golden", @@ -2548,20 +2541,20 @@ "work_dir_golden": "target/llm-runs/basics/t_006_delete/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_006_delete/csharp/server/gpt-5/llm", "scorer_details": { - "seed_users_row": { + "delete_user_count_zero": { "pass": true, "partial": 1.0, "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1" } }, - "delete_user_count_zero": { + "seed_users_row": { "pass": true, "partial": 1.0, "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1" + "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" } }, "schema_parity": { @@ -2572,25 +2565,25 @@ "llm_db": "basics-t-006-delete-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:46871", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T12:05:24.880862690Z", - "finished_at": "2026-01-23T12:06:10.897631772Z" + "started_at": "2026-01-23T20:52:51.107488271Z", + "finished_at": "2026-01-23T20:53:13.731066286Z" }, "t_007_crud": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_007_crud", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n if (ctx.Db.User.Id.Find(1) is User user1)\n {\n user1.Name = \"Alice2\";\n user1.Age = 31;\n user1.Active = false;\n ctx.Db.User.Id.Update(user1);\n }\n\n ctx.Db.User.Id.Delete(2);\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n ctx.Db.User.Id.Update(new User { Id = 1, Name = \"Alice2\", Age = 31, Active = false });\n ctx.Db.User.Id.Delete(2);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -2598,28 +2591,6 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/csharp/server/gpt-5/llm", "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-007-crud-golden", - "llm_db": "basics-t-007-crud-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:35813", - "tables_diff": null, - "tables_equal": true - } - }, "crud_row_id1_parity": { "pass": true, "partial": 1.0, @@ -2631,7 +2602,7 @@ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "Crud", - "server": "http://127.0.0.1:35813" + "server": "http://127.0.0.1:46871" } }, "crud_total_count_one": { @@ -2642,21 +2613,43 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM User" } + }, + "crud_row_id2_deleted": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-007-crud-golden", + "llm_db": "basics-t-007-crud-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:46871", + "tables_diff": null, + "tables_equal": true + } } }, "vendor": "openai", - "started_at": "2026-01-23T12:04:36.458767943Z", - "finished_at": "2026-01-23T12:05:24.618417126Z" + "started_at": "2026-01-23T20:52:08.554828637Z", + "finished_at": "2026-01-23T20:52:51.107450441Z" }, "t_008_index_lookup": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_008_index_lookup", "lang": "csharp", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n if (ctx.Db.Result.Id.Find(id) is Result existing)\n {\n existing.Name = user.Name;\n ctx.Db.Result.Id.Update(existing);\n }\n else\n {\n ctx.Db.Result.Insert(new Result { Id = user.Id, Name = user.Name });\n }\n }\n }\n}", + "passed_tests": 0, + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n var user = ctx.Db.User.Id.Find(id);\n if (user != null)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = user.Id,\n Name = user.Name\n });\n }\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -2664,56 +2657,28 @@ "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm", "scorer_details": { - "seed_user_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-008-index-lookup-golden", - "llm_db": "basics-t-008-index-lookup-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:35813", - "tables_diff": null, - "tables_equal": true - } - }, - "index_lookup_projection_parity": { - "pass": true, - "partial": 1.0, + "publish_error": { + "pass": false, + "partial": 0.0, "notes": { - "args": [ - 1 - ], - "golden_db": "basics-t-008-index-lookup-golden", - "golden_out": "Id | Name ----+--------- 1 | \"Alice\"", - "llm_db": "basics-t-008-index-lookup-gpt-5-llm", - "llm_out": "Id | Name ----+--------- 1 | \"Alice\"", - "query": "SELECT Id, Name FROM Result WHERE Id=1", - "reducer": "LookupUserName", - "server": "http://127.0.0.1:35813" + "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\n/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm/Lib.cs(32,27): error CS1061: 'Module.User?' does not contain a definition for 'Id' and no accessible extension method 'Id' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm/StdbModule.csproj]\n/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm/Lib.cs(33,29): error CS1061: 'Module.User?' does not contain a definition for 'Name' and no accessible extension method 'Name' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm/StdbModule.csproj]\n", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T12:05:24.876913581Z", - "finished_at": "2026-01-23T12:06:14.796358950Z" + "started_at": "2026-01-23T20:52:47.129209751Z", + "finished_at": "2026-01-23T20:53:08.577581818Z" }, "t_009_init": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_009_init", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"User\")]\n public partial struct User\n {\n [PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -2729,18 +2694,18 @@ "llm_db": "basics-t-009-init-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:46871", "tables_diff": null, "tables_equal": true } }, - "init_seed_alice": { + "init_seed_bob": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" } }, "init_total_two": { @@ -2752,29 +2717,29 @@ "sql": "SELECT COUNT(*) AS n FROM User" } }, - "init_seed_bob": { + "init_seed_alice": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" } } }, "vendor": "openai", - "started_at": "2026-01-23T12:05:18.162783655Z", - "finished_at": "2026-01-23T12:06:11.635711073Z" + "started_at": "2026-01-23T20:52:36.884011764Z", + "finished_at": "2026-01-23T20:53:00.062192214Z" }, "t_010_connect": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_010_connect", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Event\", Public = true)]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Kind = \"disconnected\" });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Event\", Public = true)]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"disconnected\" });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-010-connect-golden", @@ -2790,25 +2755,25 @@ "llm_db": "basics-t-010-connect-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:46871", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T12:04:30.187575150Z", - "finished_at": "2026-01-23T12:05:18.162739983Z" + "started_at": "2026-01-23T20:52:05.996450500Z", + "finished_at": "2026-01-23T20:52:36.883985146Z" }, "t_011_helper_function": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_011_helper_function", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n public static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result { Id = id, Sum = Add(a, b) });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n public static int Add(int a, int b) => a + b;\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Sum = Add(a, b)\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-011-helper-function-golden", @@ -2816,6 +2781,15 @@ "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/csharp/server/gpt-5/llm", "scorer_details": { + "helper_func_sum_abs": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1 AND Sum=5" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -2824,20 +2798,11 @@ "llm_db": "basics-t-011-helper-function-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:46871", "tables_diff": null, "tables_equal": true } }, - "helper_func_sum_abs": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1 AND Sum=5" - } - }, "helper_func_sum_parity": { "pass": true, "partial": 1.0, @@ -2853,23 +2818,23 @@ "llm_out": "Id | Sum ----+----- 1 | 5", "query": "SELECT Id, Sum FROM Result WHERE Id=1", "reducer": "ComputeSum", - "server": "http://127.0.0.1:35813" + "server": "http://127.0.0.1:46871" } } }, "vendor": "openai", - "started_at": "2026-01-23T12:03:14.890278760Z", - "finished_at": "2026-01-23T12:04:24.844053542Z" + "started_at": "2026-01-23T20:51:08.724127650Z", + "finished_at": "2026-01-23T20:52:02.544972253Z" }, "t_012_spacetime_product_type": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_012_spacetime_product_type", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Score\n {\n Left = left,\n Right = right\n }\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Score\n {\n public int Left;\n public int Right;\n }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Score Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetScore(ReducerContext ctx, int id, int left, int right)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Score { Left = left, Right = right }\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -2877,22 +2842,17 @@ "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/csharp/server/gpt-5/llm", "scorer_details": { - "product_type_row_parity": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "args": [ - 1, - 2, - 3 - ], "golden_db": "schema-t-012-spacetime-product-type-golden", - "golden_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)", "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", - "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)", - "query": "SELECT Id, Value FROM Result WHERE Id=1", - "reducer": "SetScore", - "server": "http://127.0.0.1:35813" + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:46871", + "tables_diff": null, + "tables_equal": true } }, "product_type_row_count": { @@ -2904,33 +2864,38 @@ "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" } }, - "schema_parity": { + "product_type_row_parity": { "pass": true, "partial": 1.0, "notes": { + "args": [ + 1, + 2, + 3 + ], "golden_db": "schema-t-012-spacetime-product-type-golden", + "golden_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)", "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:35813", - "tables_diff": null, - "tables_equal": true + "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)", + "query": "SELECT Id, Value FROM Result WHERE Id=1", + "reducer": "SetScore", + "server": "http://127.0.0.1:46871" } } }, "vendor": "openai", - "started_at": "2026-01-23T12:06:44.805001437Z", - "finished_at": "2026-01-23T12:07:16.896251109Z" + "started_at": "2026-01-23T20:53:40.420411949Z", + "finished_at": "2026-01-23T20:54:07.766554713Z" }, "t_013_spacetime_sum_type": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_013_spacetime_sum_type", "lang": "csharp", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n var existing = ctx.Db.Result.Id.Find(id);\n if (existing != null)\n {\n ctx.Db.Result.Id.Delete(id);\n }\n\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = new Shape.Circle(new Circle { Radius = radius })\n });\n }\n}", + "passed_tests": 0, + "llm_output": "using SpacetimeDB;\nusing SpacetimeDB.Types;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n var shape = new Shape.Circle(new Circle { Radius = radius });\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = shape\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -2938,52 +2903,21 @@ "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/csharp/server/gpt-5/llm", "scorer_details": { - "sum_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 10 - ], - "golden_db": "schema-t-013-spacetime-sum-type-golden", - "golden_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))", - "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", - "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))", - "query": "SELECT Id, Value FROM Result WHERE Id=1", - "reducer": "SetCircle", - "server": "http://127.0.0.1:35813" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-013-spacetime-sum-type-golden", - "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:35813", - "tables_diff": null, - "tables_equal": true - } - }, - "sum_type_row_count": { - "pass": true, - "partial": 1.0, + "publish_error": { + "pass": false, + "partial": 0.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" + "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\n/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_013_spacetime_sum_type/csharp/server/gpt-5/llm/Lib.cs(3,19): error CS0234: The type or namespace name 'Types' does not exist in the namespace 'SpacetimeDB' (are you missing an assembly reference?) [/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_013_spacetime_sum_type/csharp/server/gpt-5/llm/StdbModule.csproj]\n", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T12:07:16.896327053Z", - "finished_at": "2026-01-23T12:08:07.936339455Z" + "started_at": "2026-01-23T20:54:05.268678013Z", + "finished_at": "2026-01-23T20:54:43.846818259Z" }, "t_014_elementary_columns": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_014_elementary_columns", "lang": "csharp", "golden_published": true, @@ -2998,49 +2932,49 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, + "elementary_columns_row_count": { + "pass": false, + "partial": 0.0, "notes": { - "golden_db": "schema-t-014-elementary-columns-golden", - "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:35813", - "tables_diff": null, - "tables_equal": true + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c2003883ecec05b675876359540c06439933bae20ff47c4c334ae2cfa397c954/sql)\n", + "phase": "sql" } }, "elementary_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35813/v1/database/c20099d4196f0168d509841ccbf52a41e879a5de183a360f652419198a45fa09/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c200ba25315ff7445dc543e2e42a2309cd493cfe1974181f53e50fa06c464134/sql)\n", "phase": "sql_golden" } }, - "elementary_columns_row_count": { - "pass": false, - "partial": 0.0, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35813/v1/database/c20009038c3974b6dd9b27cd833b5bf2548c34fdbd6898aac40238455b2a0d9b/sql)\n", - "phase": "sql" + "golden_db": "schema-t-014-elementary-columns-golden", + "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:46871", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T12:06:54.360546232Z", - "finished_at": "2026-01-23T12:07:34.648602066Z" + "started_at": "2026-01-23T20:53:43.824629255Z", + "finished_at": "2026-01-23T20:54:14.326210872Z" }, "t_015_product_type_columns": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_015_product_type_columns", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Profile\", Public = true)]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Profile.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 }\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Profile\", Public = true)]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var row in ctx.Db.Profile.Iter())\n {\n ctx.Db.Profile.Id.Delete(row.Id);\n }\n\n ctx.Db.Profile.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 }\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -3048,20 +2982,6 @@ "work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/csharp/server/gpt-5/llm", "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_db": "schema-t-015-product-type-columns-golden", - "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", - "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", - "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", - "query": "SELECT Id, Home, Work, Pos FROM Profile WHERE Id=1", - "reducer": "Seed", - "server": "http://127.0.0.1:35813" - } - }, "product_type_columns_row_count": { "pass": true, "partial": 1.0, @@ -3079,25 +2999,39 @@ "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:46871", "tables_diff": null, "tables_equal": true } + }, + "product_type_columns_row_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [], + "golden_db": "schema-t-015-product-type-columns-golden", + "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", + "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", + "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", + "query": "SELECT Id, Home, Work, Pos FROM Profile WHERE Id=1", + "reducer": "Seed", + "server": "http://127.0.0.1:46871" + } } }, "vendor": "openai", - "started_at": "2026-01-23T12:07:16.899960622Z", - "finished_at": "2026-01-23T12:07:53.777092694Z" + "started_at": "2026-01-23T20:54:07.846975920Z", + "finished_at": "2026-01-23T20:54:41.128204875Z" }, "t_016_sum_type_columns": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_016_sum_type_columns", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Drawing\", Public = true)]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Drawing.Insert(new Drawing\n {\n Id = 1,\n A = new Shape.Circle(new Circle { Radius = 10 }),\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Drawing\")]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n var a = new Shape.Circle(new Circle { Radius = 10 });\n var b = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 });\n ctx.Db.Drawing.Insert(new Drawing { Id = 1, A = a, B = b });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -3105,14 +3039,6 @@ "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm", "scorer_details": { - "sum_type_columns_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35813/v1/database/c200b0343bd80e3f58914be7679e885cb40098290c8573df5d5e85bf5f1d8191/sql)\n", - "phase": "sql_golden" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -3121,33 +3047,41 @@ "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:46871", "tables_diff": null, "tables_equal": true } }, + "sum_type_columns_row_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c2002b630c1898fdca0d5df967295e9965d354ea89c64878bb9c71421ac210de/sql)\n", + "phase": "sql_golden" + } + }, "sum_type_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35813/v1/database/c200ad3cc29488a8fe57b5ab9ac0432417ab7d585d48e65d407028387050d856/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c200255f75652e874ebf03622fd9c2dfb3b65b42f575d4fa5a95cc36e4182f25/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-23T12:07:31.832777595Z", - "finished_at": "2026-01-23T12:08:22.217518757Z" + "started_at": "2026-01-23T20:54:07.849642740Z", + "finished_at": "2026-01-23T20:54:53.203074499Z" }, "t_017_scheduled_columns": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_017_scheduled_columns", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 2, "passed_tests": 1, - "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer _timer)\n {\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromTicks(50_000 * 10))\n });\n }\n}", + "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer row)\n {\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledId = 0,\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromTicks(500_000))\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -3155,14 +3089,6 @@ "work_dir_golden": "target/llm-runs/schema/t_017_scheduled_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_017_scheduled_columns/csharp/server/gpt-5/llm", "scorer_details": { - "scheduled_seeded_one_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35813/v1/database/c200805f92c3324ddd49c14c46faf1bfa5b52a3ebae1e3664df720fbd9d4a487/sql)\n", - "phase": "sql" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -3171,18 +3097,26 @@ "llm_db": "schema-t-017-scheduled-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:46871", "tables_diff": null, "tables_equal": true } + }, + "scheduled_seeded_one_row": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c200bcc4292b7995634fb91d48b6baa5c9096e7f85c4b73e7db1b4f1f377a348/sql)\n", + "phase": "sql" + } } }, "vendor": "openai", - "started_at": "2026-01-23T12:06:40.441707268Z", - "finished_at": "2026-01-23T12:07:31.832707547Z" + "started_at": "2026-01-23T20:53:28.130925948Z", + "finished_at": "2026-01-23T20:54:07.846947117Z" }, "t_018_constraints": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_018_constraints", "lang": "csharp", "golden_published": true, @@ -3197,19 +3131,6 @@ "work_dir_golden": "target/llm-runs/schema/t_018_constraints/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_018_constraints/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-018-constraints-golden", - "llm_db": "schema-t-018-constraints-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:35813", - "tables_diff": null, - "tables_equal": true - } - }, "constraints_row_parity_after_seed": { "pass": true, "partial": 1.0, @@ -3221,7 +3142,7 @@ "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", "query": "SELECT Id, Email, Name FROM Account WHERE Id=1", "reducer": "Seed", - "server": "http://127.0.0.1:35813" + "server": "http://127.0.0.1:46871" } }, "constraints_seed_two_rows": { @@ -3232,21 +3153,34 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM Account WHERE Id=2" } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-018-constraints-golden", + "llm_db": "schema-t-018-constraints-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:46871", + "tables_diff": null, + "tables_equal": true + } } }, "vendor": "openai", - "started_at": "2026-01-23T12:06:11.639612541Z", - "finished_at": "2026-01-23T12:06:44.804960222Z" + "started_at": "2026-01-23T20:53:08.577602895Z", + "finished_at": "2026-01-23T20:53:40.420363798Z" }, "t_019_many_to_many": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_019_many_to_many", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 5, "passed_tests": 5, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"Group\")]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"Membership\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { \"UserId\" })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { \"GroupId\" })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(SpacetimeDB.ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"Group\", Public = true)]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"Membership\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { \"UserId\" })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { \"GroupId\" })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var m in ctx.Db.Membership.Iter())\n {\n ctx.Db.Membership.Id.Delete(m.Id);\n }\n foreach (var g in ctx.Db.Group.Iter())\n {\n ctx.Db.Group.GroupId.Delete(g.GroupId);\n }\n foreach (var u in ctx.Db.User.Iter())\n {\n ctx.Db.User.UserId.Delete(u.UserId);\n }\n\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-019-many-to-many-golden", @@ -3254,13 +3188,17 @@ "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm", "scorer_details": { - "m2m_has_1_10": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=10" + "golden_db": "schema-t-019-many-to-many-golden", + "llm_db": "schema-t-019-many-to-many-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:46871", + "tables_diff": null, + "tables_equal": true } }, "memberships_three_rows": { @@ -3272,26 +3210,22 @@ "sql": "SELECT COUNT(*) AS n FROM Membership" } }, - "schema_parity": { + "m2m_has_2_20": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "schema-t-019-many-to-many-golden", - "llm_db": "schema-t-019-many-to-many-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:35813", - "tables_diff": null, - "tables_equal": true + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=2 AND GroupId=20" } }, - "m2m_has_2_20": { + "m2m_has_1_10": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=2 AND GroupId=20" + "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=10" } }, "m2m_has_1_20": { @@ -3305,18 +3239,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T12:06:14.796478395Z", - "finished_at": "2026-01-23T12:07:15.676945464Z" + "started_at": "2026-01-23T20:53:13.731095349Z", + "finished_at": "2026-01-23T20:54:05.268633334Z" }, "t_020_ecs": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_020_ecs", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 5, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Entity\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"Position\", Public = true)]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Velocity\", Public = true)]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"NextPosition\", Public = true)]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n // Entity 1\n if (ctx.Db.Entity.Id.Find(1) is not Entity e1)\n {\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n }\n if (ctx.Db.Position.EntityId.Find(1) is not Position p1)\n {\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n }\n if (ctx.Db.Velocity.EntityId.Find(1) is not Velocity v1)\n {\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n }\n\n // Entity 2\n if (ctx.Db.Entity.Id.Find(2) is not Entity e2)\n {\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n }\n if (ctx.Db.Position.EntityId.Find(2) is not Position p2)\n {\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n }\n if (ctx.Db.Velocity.EntityId.Find(2) is not Velocity v2)\n {\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.Position.Iter())\n {\n if (ctx.Db.Velocity.EntityId.Find(pos.EntityId) is not Velocity vel)\n {\n continue;\n }\n\n int nx = pos.X + vel.VX;\n int ny = pos.Y + vel.VY;\n\n if (ctx.Db.NextPosition.EntityId.Find(pos.EntityId) is NextPosition np)\n {\n np.X = nx;\n np.Y = ny;\n ctx.Db.NextPosition.EntityId.Update(np);\n }\n else\n {\n ctx.Db.NextPosition.Insert(new NextPosition\n {\n EntityId = pos.EntityId,\n X = nx,\n Y = ny\n });\n }\n }\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Entity\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"Position\", Public = true)]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Velocity\", Public = true)]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"NextPosition\", Public = true)]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n // Entity 1\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n\n // Entity 2\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.Position.Iter())\n {\n var vel = ctx.Db.Velocity.EntityId.Find(pos.EntityId);\n if (vel == null)\n {\n continue;\n }\n\n int nextX = pos.X + vel.Value.VX;\n int nextY = pos.Y + vel.Value.VY;\n\n var existing = ctx.Db.NextPosition.EntityId.Find(pos.EntityId);\n if (existing != null)\n {\n var np = existing.Value;\n np.X = nextX;\n np.Y = nextY;\n ctx.Db.NextPosition.EntityId.Update(np);\n }\n else\n {\n ctx.Db.NextPosition.Insert(new NextPosition\n {\n EntityId = pos.EntityId,\n X = nextX,\n Y = nextY\n });\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -3324,19 +3258,11 @@ "work_dir_golden": "target/llm-runs/schema/t_020_ecs/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm", "scorer_details": { - "ecs_seed_positions_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35813/v1/database/c2007f4d2ed8ff707a6d253e582ebc322ea6e6e956e5d4216cd59b8039172aaf/sql)\n", - "phase": "sql" - } - }, - "ecs_next_pos_entity2": { + "ecs_next_pos_entity1": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35813/v1/database/c2007f4d2ed8ff707a6d253e582ebc322ea6e6e956e5d4216cd59b8039172aaf/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c20001cd05b3f1e81ac991a3d37765e210ae358de6cdec43d4918b54b938eee1/sql)\n", "phase": "sql" } }, @@ -3348,41 +3274,49 @@ "llm_db": "schema-t-020-ecs-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:46871", "tables_diff": null, "tables_equal": true } }, + "ecs_next_pos_entity2": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c20001cd05b3f1e81ac991a3d37765e210ae358de6cdec43d4918b54b938eee1/sql)\n", + "phase": "sql" + } + }, "ecs_step_next_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35813/v1/database/c2007f4d2ed8ff707a6d253e582ebc322ea6e6e956e5d4216cd59b8039172aaf/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c20001cd05b3f1e81ac991a3d37765e210ae358de6cdec43d4918b54b938eee1/sql)\n", "phase": "sql" } }, - "ecs_next_pos_entity1": { + "ecs_seed_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:35813/v1/database/c2007f4d2ed8ff707a6d253e582ebc322ea6e6e956e5d4216cd59b8039172aaf/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c20001cd05b3f1e81ac991a3d37765e210ae358de6cdec43d4918b54b938eee1/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-23T12:05:47.466492129Z", - "finished_at": "2026-01-23T12:06:40.441631392Z" + "started_at": "2026-01-23T20:52:53.389850738Z", + "finished_at": "2026-01-23T20:53:43.824579798Z" }, "t_021_multi_column_index": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "task": "t_021_multi_column_index", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Log\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = new[] { nameof(UserId), nameof(Day) })]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var row in ctx.Db.Log.Iter())\n {\n ctx.Db.Log.Id.Delete(row.Id);\n }\n\n ctx.Db.Log.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.Log.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.Log.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Log\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = new[] { \"UserId\", \"Day\" })]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Log.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.Log.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.Log.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -3407,33 +3341,33 @@ "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:35813", + "server": "http://127.0.0.1:46871", "tables_diff": null, "tables_equal": true } }, - "mcindex_seed_count": { + "mcindex_lookup_u7_d2": { "pass": true, "partial": 1.0, "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Log" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=2" } }, - "mcindex_lookup_u7_d2": { + "mcindex_seed_count": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=2" + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM Log" } } }, "vendor": "openai", - "started_at": "2026-01-23T12:06:11.635827651Z", - "finished_at": "2026-01-23T12:06:54.360432909Z" + "started_at": "2026-01-23T20:53:00.062219949Z", + "finished_at": "2026-01-23T20:53:28.130894765Z" } } } diff --git a/docs/llms/docs-benchmark-summary.json b/docs/llms/docs-benchmark-summary.json index f5405f61c51..8fd73afcfd8 100644 --- a/docs/llms/docs-benchmark-summary.json +++ b/docs/llms/docs-benchmark-summary.json @@ -1,38 +1,38 @@ { "version": 1, - "generated_at": "2026-01-23T12:08:22.290Z", + "generated_at": "2026-01-23T20:54:53.246Z", "by_language": { "csharp": { "modes": { "docs": { - "hash": "8d9524a0b2f5ad54b0364657a5d8eca4c9ccdcec9c7c83a41666c57e15b0af0e", + "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", "models": { "GPT-5": { "categories": { "basics": { "tasks": 12, "total_tests": 27, - "passed_tests": 27, - "pass_pct": 100.0, - "task_pass_equiv": 12.0, - "task_pass_pct": 100.0 + "passed_tests": 24, + "pass_pct": 88.888885, + "task_pass_equiv": 11.0, + "task_pass_pct": 91.66667 }, "schema": { "tasks": 10, "total_tests": 34, - "passed_tests": 25, - "pass_pct": 73.52941, - "task_pass_equiv": 7.3666663, - "task_pass_pct": 73.666664 + "passed_tests": 22, + "pass_pct": 64.70588, + "task_pass_equiv": 6.366667, + "task_pass_pct": 63.666664 } }, "totals": { "tasks": 22, "total_tests": 61, - "passed_tests": 52, - "pass_pct": 85.2459, - "task_pass_equiv": 19.366667, - "task_pass_pct": 88.030304 + "passed_tests": 46, + "pass_pct": 75.409836, + "task_pass_equiv": 17.366667, + "task_pass_pct": 78.9394 } } } @@ -42,7 +42,7 @@ "rust": { "modes": { "docs": { - "hash": "b31ac877c642ddd2969fd1af34cc203a2d3232b9e63b82fceacaf51212a600b9", + "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", "models": { "GPT-5": { "categories": { @@ -56,20 +56,20 @@ }, "schema": { "tasks": 10, - "total_tests": 34, - "passed_tests": 8, - "pass_pct": 23.529411, - "task_pass_equiv": 2.05, - "task_pass_pct": 20.5 + "total_tests": 30, + "passed_tests": 4, + "pass_pct": 13.333333, + "task_pass_equiv": 1.25, + "task_pass_pct": 12.5 } }, "totals": { "tasks": 22, - "total_tests": 61, - "passed_tests": 13, - "pass_pct": 21.311476, - "task_pass_equiv": 3.3833334, - "task_pass_pct": 15.378788 + "total_tests": 57, + "passed_tests": 9, + "pass_pct": 15.789474, + "task_pass_equiv": 2.5833335, + "task_pass_pct": 11.742425 } } } @@ -90,19 +90,19 @@ "schema": { "tasks": 10, "total_tests": 34, - "passed_tests": 23, - "pass_pct": 67.64706, - "task_pass_equiv": 6.533333, - "task_pass_pct": 65.33333 + "passed_tests": 26, + "pass_pct": 76.47059, + "task_pass_equiv": 7.5333333, + "task_pass_pct": 75.333336 } }, "totals": { "tasks": 22, "total_tests": 61, - "passed_tests": 45, - "pass_pct": 73.77049, - "task_pass_equiv": 15.45, - "task_pass_pct": 70.22727 + "passed_tests": 48, + "pass_pct": 78.68852, + "task_pass_equiv": 16.45, + "task_pass_pct": 74.772736 } } } From f49f69d6e77f095dc9a668eee26659e96ab2b885 Mon Sep 17 00:00:00 2001 From: clockwork-labs-bot Date: Sat, 24 Jan 2026 00:06:38 +0000 Subject: [PATCH 4/4] Update LLM benchmark results --- docs/llms/docs-benchmark-analysis.md | 519 ++++--- docs/llms/docs-benchmark-comment.md | 18 +- docs/llms/docs-benchmark-details.json | 1787 +++++++++++++------------ docs/llms/docs-benchmark-summary.json | 62 +- 4 files changed, 1265 insertions(+), 1121 deletions(-) diff --git a/docs/llms/docs-benchmark-analysis.md b/docs/llms/docs-benchmark-analysis.md index 17b5c26d78c..f1ac99a24bc 100644 --- a/docs/llms/docs-benchmark-analysis.md +++ b/docs/llms/docs-benchmark-analysis.md @@ -4,251 +4,370 @@ Generated from: `/__w/SpacetimeDB/SpacetimeDB/tools/xtask-llm-benchmark/../../do ## Summary -- **Total failures analyzed**: 36 +- **Total failures analyzed**: 35 --- # SpacetimeDB Benchmark Failures Analysis -This document analyzes test failures in the SpacetimeDB benchmark organized by language and mode. For each failure, we provide the generated code, the expected code, the error message, and a detailed explanation along with actionable recommendations. +## Rust / rustdoc_json Failures -## Rust / rustdoc_json Failures (8 total) +### Compile/Publish Errors -### Compile/Publish Errors (2 failures) +#### t_002_scheduled_table +**The generated code:** +```rust +use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt}; -#### t_002_scheduled_table & t_017_scheduled_columns +#[table(name = tick_timer, scheduled(reducer = tick, column = scheduled_at))] +pub struct TickTimer { + #[primary_key] + #[auto_inc] + scheduled_id: u64, + scheduled_at: ScheduleAt, +} -1. **The generated code**: - ```rust - use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt}; - - #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))] - pub struct TickTimer { - #[primary_key] - #[auto_inc] - scheduled_id: u64, - scheduled_at: ScheduleAt, - } - - #[reducer(init)] - pub fn init(ctx: &ReducerContext) { - if ctx.db.tick_timer().count() == 0 { - ctx.db.tick_timer().insert(TickTimer { - scheduled_id: 0, - scheduled_at: ScheduleAt::repeat(std::time::Duration::from_micros(50_000)), - }); - } - } - - #[reducer] - pub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {} - ``` - -2. **The expected code**: - ```rust - use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; - use std::time::Duration; - - #[table(name = tick_timer, scheduled(tick))] - pub struct TickTimer { - #[primary_key] - #[auto_inc] - pub scheduled_id: u64, - pub scheduled_at: ScheduleAt, - } - - #[reducer] - pub fn tick(_ctx: &ReducerContext, _schedule: TickTimer) {} - - #[reducer(init)] - pub fn init(ctx: &ReducerContext) { - let every_50ms: ScheduleAt = Duration::from_millis(50).into(); +#[reducer(init)] +pub fn init(ctx: &ReducerContext) { + if ctx.db.tick_timer().count() == 0 { ctx.db.tick_timer().insert(TickTimer { scheduled_id: 0, - scheduled_at: every_50ms, + scheduled_at: ScheduleAt::repeat(50_000u64), }); } - ``` - -3. **The error**: `publish_error: spacetime publish failed (exit=1)` - -4. **Explain the difference**: - - Incorrect `#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]` should use `#[table(name = tick_timer, scheduled(tick))]` - - Use of `ScheduleAt::repeat` is incorrect; it should use a proper time duration constructor. - -5. **Root cause**: The documentation lacks clarity on scheduling syntax and constructors for time intervals in scheduled tasks. - -6. **Recommendation**: Update documentation to emphasize using `scheduled(tick)` and correct constructors for `ScheduleAt` using `Duration::from_millis`. - ---- - -### Other Failures (6 failures) - -#### t_003_struct_in_table, t_004_insert, t_007_crud, t_011_helper_function, t_016_sum_type_columns - -1. **The generated code** (e.g., for `t_003`): - ```rust - use spacetimedb::{ReducerContext, Table, UniqueColumn, SpacetimeType}; - - #[derive(SpacetimeType, Clone)] - pub struct Position { - pub x: i32, - pub y: i32, - } - - #[spacetimedb::table(name = entity)] - pub struct Entity { - #[primary_key] - pub id: i32, - pub pos: Position, - } - ``` - -2. **The expected code**: - ```rust - use spacetimedb::{table, SpacetimeType}; - - #[derive(SpacetimeType, Clone, Debug)] - pub struct Position { - pub x: i32, - pub y: i32, - } - - #[table(name = entity)] - pub struct Entity { - #[primary_key] - pub id: i32, - pub pos: Position, +} + +#[reducer] +pub fn tick(_ctx: &ReducerContext, _scheduled_id: u64) { +} +``` + +**The golden example:** +```rust +use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; +use std::time::Duration; + +#[table(name = tick_timer, scheduled(tick))] +pub struct TickTimer { + #[primary_key] + #[auto_inc] + pub scheduled_id: u64, + pub scheduled_at: ScheduleAt, +} + +#[reducer] +pub fn tick(_ctx: &ReducerContext, _row: TickTimer) -> Result<(), String> { + Ok(()) +} + +#[reducer(init)] +pub fn init(ctx: &ReducerContext) -> Result<(), String> { + ctx.db.tick_timer().insert(TickTimer { + scheduled_id: 0, + scheduled_at: ScheduleAt::Interval(Duration::from_millis(50).into()), + }); + Ok(()) +} +``` + +**The error:** `publish_error: spacetime publish failed (exit=1)` + +**Explain the difference:** +1. The LLM generated code uses incorrect API methods and structures. For instance, it uses `ScheduleAt::repeat(50_000u64)` instead of the correct `ScheduleAt::Interval(Duration::from_millis(50).into())`. +2. The reducer function `tick` should also return a `Result<(), String>` type as in the golden example. + +**Root cause:** The documentation may lack clarity on the specific types and return values required, especially around scheduling and reducers. + +**Recommendation:** +Update documentation to clarify the expected types and return values. Use the following: +- Make sure to specify correct API functions and structures in examples. + +#### t_017_scheduled_columns +**The generated code:** +```rust +use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt}; + +#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))] +pub struct TickTimer { + #[primary_key] + #[auto_inc] + scheduled_id: u64, + scheduled_at: ScheduleAt, +} + +#[reducer(init)] +pub fn init(ctx: &ReducerContext) { + let tbl = ctx.db.tick_timer(); + if tbl.count() == 0 { + tbl.insert(TickTimer { + scheduled_id: 0, + scheduled_at: ScheduleAt::RepeatMicros(50_000), + }); } - ``` +} + +#[reducer] +pub fn tick(_ctx: &ReducerContext, _row: TickTimer) { +} +``` + +**The golden example:** +```rust +use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; +use std::time::Duration; + +#[table(name = tick_timer, scheduled(tick))] +pub struct TickTimer { + #[primary_key] + #[auto_inc] + pub scheduled_id: u64, + pub scheduled_at: ScheduleAt, +} + +#[reducer] +pub fn tick(_ctx: &ReducerContext, _schedule: TickTimer) { +} + +#[reducer(init)] +pub fn init(ctx: &ReducerContext) { + let every_50ms: ScheduleAt = Duration::from_millis(50).into(); + ctx.db.tick_timer().insert(TickTimer { + scheduled_id: 0, + scheduled_at: every_50ms, + }); +} +``` + +**The error:** `publish_error: spacetime publish failed (exit=1)` + +**Explain the difference:** +1. The LLM incorrectly uses an outdated or wrong structure for scheduling with `ScheduleAt::RepeatMicros(50_000)`. + +**Root cause:** There is likely a mismatch in expected vs. generated syntax, particularly around the scheduling configuration settings. + +**Recommendation:** +Reiterate the correct usage of scheduling in the updated documentation with ample examples. -3. **The error**: `schema_parity: reducers differ - expected [], got [...]` - -4. **Explain the difference**: Missing `pub` for fields in structs which are not public, causing access issues. - -5. **Root cause**: Insufficient detail in documentation about struct visibility and reducing/scheduling attributes. +--- -6. **Recommendation**: Clarify that public fields are required for structs defining database tables. +### Other Failures + +#### t_001_basic_tables +**The generated code:** +```rust +use spacetimedb::ReducerContext; + +#[spacetimedb::table(name = user)] +pub struct User { + #[primary_key] + id: i32, + name: String, + age: i32, + active: bool, +} + +#[spacetimedb::table(name = product)] +pub struct Product { + #[primary_key] + id: i32, + title: String, + price: f32, + in_stock: bool, +} + +#[spacetimedb::table(name = note)] +pub struct Note { + #[primary_key] + id: i32, + body: String, + rating: i64, + pinned: bool, +} + +#[spacetimedb::reducer(init)] +pub fn init(_ctx: &ReducerContext) {} +``` + +**The golden example:** +```rust +use spacetimedb::table; + +#[table(name = user)] +pub struct User { + #[primary_key] + pub id: i32, + pub name: String, + pub age: i32, + pub active: bool, +} + +#[table(name = product)] +pub struct Product { + #[primary_key] + pub id: i32, + pub title: String, + pub price: f32, + pub in_stock: bool, +} + +#[table(name = note)] +pub struct Note { + #[primary_key] + pub id: i32, + pub body: String, + pub rating: i64, + pub pinned: bool, +} +``` + +**The error:** `schema_parity: reducers differ - expected [], got ["init()"]` + +**Explain the difference:** The generated code does not include any public fields in the struct definitions, which is a requirement for proper table mapping. + +**Root cause:** Public visibility for struct fields may not have been emphasized in the existing documentation. + +**Recommendation:** +Explain the importance of making struct fields public in documentation examples to prevent mismatches during schema checks. --- -#### Additional Observations: - -- The focus must be on both visibility modifiers and correct API signatures for reducers and tables. -- The need for `Result<(), String>` in reducer functions is missing in many generated snippets. +### Insert and CRUD Related Tests + +#### t_004_insert +**The generated code:** +```rust +use spacetimedb::{reducer, table, ReducerContext, Table}; + +#[table(name = user)] +pub struct User { + #[primary_key] + id: i32, + name: String, + age: i32, + active: bool, +} + +#[reducer] +pub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) { + ctx.db.user().insert(User { id, name, age, active }); +} +``` + +**The golden example:** +```rust +use spacetimedb::{reducer, table, ReducerContext, Table}; + +#[table(name = user)] +pub struct User { + #[primary_key] + pub id: i32, + pub name: String, + pub age: i32, + pub active: bool, +} + +#[reducer] +pub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) -> Result<(), String> { + ctx.db.user().insert(User { id, name, age, active }); + Ok(()) +} +``` + +**The error:** `data_parity_insert_user: spacetime sql failed: Error: no such table: user.` + +**Explain the difference:** Missing the `pub` visibility specifier for struct fields and failing to return a result type in reducer functions. + +**Root cause:** Lack of emphasis on the need for public struct fields in the documentation and clear result type specifications. + +**Recommendation:** +Strengthen language around public field requirements and return types in reducer functions in the documentation. --- -## Rust / docs Failures (22 total) +## C# / docs Failures -### Other Failures (22 failures) +### Timeout Issues -#### t_000_empty_reducers, t_001_basic_tables, t_002_scheduled_table, t_004_insert +#### t_020_ecs +*This failure is expected in both Rust and C#, highlighting a systemic timeout issue.* -1. **The generated code** (for `t_000_empty_reducers`): - ```rust - use spacetimedb::ReducerContext; +--- - #[spacetimedb::reducer] - pub fn empty_reducer_no_args(_ctx: &ReducerContext) {} - ``` +### Other Failures -2. **The expected code**: - ```rust - use spacetimedb::{reducer, ReducerContext}; +#### t_002_scheduled_table +**The generated code:** +```csharp +using SpacetimeDB; - #[reducer] - pub fn empty_reducer_no_args(ctx: &ReducerContext) -> Result<(), String> { - Ok(()) +public static partial class Module +{ + [Table(Name = "TickTimer", Scheduled = nameof(Tick), ScheduledAt = nameof(TickTimer.ScheduledAt))] + public partial struct TickTimer + { + [PrimaryKey, AutoInc] public ulong ScheduledId; + public ScheduleAt ScheduledAt; } - ``` - -3. **The error**: `schema_parity: describe failed: WARNING: This command is UNSTABLE` - -4. **Explain the difference**: Missing return type `Result<(), String>` for all reducer functions causes the failure. - -5. **Root cause**: Documentation does not clearly specify that all reducer functions must return a `Result`. -6. **Recommendation**: Update the documentation to explicitly require a `Result` return type for all reducer functions to avoid compilation errors. + [Reducer] + public static void Tick(ReducerContext ctx, TickTimer timer) { } ---- - -## C# / docs Failures (6 total) - -### Other Failures (6 failures) - -#### t_008_index_lookup, t_013_spacetime_sum_type + [Reducer(ReducerKind.Init)] + public static void Init(ReducerContext ctx) + { + var interval = new TimeDuration { Microseconds = 50_000 }; + ctx.Db.TickTimer.Insert(new TickTimer + { + ScheduledAt = new ScheduleAt.Interval(interval) + }); + } +} +``` -1. **The generated code** (for `t_008_index_lookup`): - ```csharp - using SpacetimeDB; +**The expected example:** +```csharp +using SpacetimeDB; - public static partial class Module +public static partial class Module +{ + [Table(Name = "TickTimer", Scheduled = nameof(Tick), ScheduledAt = nameof(TickTimer.ScheduledAt))] + public partial struct TickTimer { - [SpacetimeDB.Table(Name = "User")] - public partial struct User - { - [SpacetimeDB.PrimaryKey] - public int Id; - public string Name; - public int Age; - public bool Active; - } - - [SpacetimeDB.Reducer] - public static void LookupUserName(ReducerContext ctx, int id) - { - var user = ctx.Db.User.Id.Find(id); - if (user != null) - { - ctx.Db.Result.Insert(new Result - { - Id = user.Id, - Name = user.Name - }); - } - } + [PrimaryKey, AutoInc] public ulong ScheduledId; + public ScheduleAt ScheduledAt; } - ``` -2. **The expected code**: - ```csharp - using SpacetimeDB; + [Reducer] + public static void Tick(ReducerContext ctx, TickTimer _row) { } - public static partial class Module + [Reducer(ReducerKind.Init)] + public static void Init(ReducerContext ctx) { - [Table(Name = "User")] - public partial struct User - { - [PrimaryKey] public int Id; - public string Name; - public int Age; - public bool Active; - } - - [Reducer] - public static void LookupUserName(ReducerContext ctx, int id) + var interval = new TimeDuration { Microseconds = 50_000 }; + ctx.Db.TickTimer.Insert(new TickTimer { - var u = ctx.Db.User.Id.Find(id); - if (u.HasValue) - { - var row = u.Value; - ctx.Db.Result.Insert(new Result { Id = row.Id, Name = row.Name }); - } - } + ScheduledId = 0, + ScheduledAt = new ScheduleAt.Interval(interval) + }); } - ``` +} +``` -3. **The error**: `publish_error: spacetime build (csharp) failed (exit=1)` +**The error:** `publish_error: 500 Internal Server Error` -4. **Explain the difference**: Use of `user != null` instead of checking `u.HasValue`, which is necessary for nullable types. +**Explain the difference:** In the generated code, the scheduled fields are improperly initialized, and the reducer method signatures do not entirely match expectations regarding types and method names. -5. **Root cause**: Lacking examples for nullable types or option types in the given context. +**Root cause:** Possibly unclear examples in the documentation regarding the scheduled fields and method signature conventions. -6. **Recommendation**: Address nullable type usage in the documentation, emphasizing how to correctly check for value presence. +**Recommendation:** +Revise documentation examples to ensure all aspects of the scheduled field implementations and reducer methods are included, particularly concerning required fields. --- -### Final Thoughts +### Conclusion +The main failures across languages predominantly stem from three issues: +1. Lack of clarity on public struct field visibility. +2. Missing result type requirements in reducer methods. +3. Incorrect method signatures and scheduling configurations. -A thorough review of generator patterns and failure analysis indicates that clarifying visibility, return types, syntax for scheduling, and handling nullable types are crucial improvements for development efficiency and error avoidance in SpacetimeDB. Documenting common patterns and providing clear guidelines will enhance user experience and reduce test failures. +Specific recommendations center around **enhancing documentation with clearer examples**, **emphasizing visibility requirements**, and detailing the **return types for functions**. This will help streamline the development process and mitigate these common errors in future benchmarks. diff --git a/docs/llms/docs-benchmark-comment.md b/docs/llms/docs-benchmark-comment.md index 6a2bcda9a11..d42a62c2d60 100644 --- a/docs/llms/docs-benchmark-comment.md +++ b/docs/llms/docs-benchmark-comment.md @@ -3,15 +3,15 @@ | Language | Mode | Category | Tests Passed | Task Pass % | |----------|------|----------|--------------|-------------| | Rust | rustdoc_json | basics | 22/27 | 74.3% | -| Rust | rustdoc_json | schema | 26/34 | 75.3% ⬆️ +10.0% | -| Rust | rustdoc_json | **total** | 48/61 | **74.8%** ⬆️ +4.5% | -| Rust | docs | basics | 5/27 | 11.1% | -| Rust | docs | schema | 4/30 | 12.5% ⬇️ -8.0% | -| Rust | docs | **total** | 9/57 | **11.7%** ⬇️ -3.6% | -| C# | docs | basics | 24/27 | 91.7% ⬇️ -8.3% | -| C# | docs | schema | 22/34 | 63.7% ⬇️ -10.0% | -| C# | docs | **total** | 46/61 | **78.9%** ⬇️ -9.1% | +| Rust | rustdoc_json | schema | 26/34 | 75.3% | +| Rust | rustdoc_json | **total** | 48/61 | **74.8%** | +| Rust | docs | basics | 4/24 | 9.0% ⬇️ -2.1% | +| Rust | docs | schema | 8/34 | 20.5% ⬆️ +8.0% | +| Rust | docs | **total** | 12/58 | **14.2%** ⬆️ +2.5% | +| C# | docs | basics | 26/27 | 91.7% | +| C# | docs | schema | 24/30 | 71.7% ⬆️ +8.0% | +| C# | docs | **total** | 50/57 | **82.6%** ⬆️ +3.6% | _Compared against master branch baseline_ -Generated at: 2026-01-23T20:54:53.246Z +Generated at: 2026-01-24T00:05:38.779Z diff --git a/docs/llms/docs-benchmark-details.json b/docs/llms/docs-benchmark-details.json index 69aaf946ba3..4982ed6acb5 100644 --- a/docs/llms/docs-benchmark-details.json +++ b/docs/llms/docs-benchmark-details.json @@ -5,21 +5,21 @@ "modes": [ { "mode": "rustdoc_json", - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "models": [ { "name": "GPT-5", "route_api_model": "gpt-5", "tasks": { "t_000_empty_reducers": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_000_empty_reducers", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, count: i32) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, count: i32, name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, active: bool, ratio: f32, label: String) {\n}", + "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, _count: i32) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, _name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, _count: i32, _name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, _active: bool, _ratio: f32, _label: String) {\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-000-empty-reducers-golden", @@ -35,25 +35,25 @@ "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34477", + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:45:58.319507359Z", - "finished_at": "2026-01-23T20:46:21.669318285Z" + "started_at": "2026-01-23T23:54:12.885499372Z", + "finished_at": "2026-01-23T23:54:29.927523908Z" }, "t_001_basic_tables": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_001_basic_tables", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, - "passed_tests": 1, - "llm_output": "#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", + "passed_tests": 0, + "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(_ctx: &ReducerContext) {}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-001-basic-tables-golden", @@ -62,32 +62,37 @@ "work_dir_llm": "target/llm-runs/basics/t_001_basic_tables/rust/server/gpt-5/llm", "scorer_details": { "schema_parity": { - "pass": true, - "partial": 1.0, + "pass": false, + "partial": 0.0, "notes": { "golden_db": "basics-t-001-basic-tables-golden", "llm_db": "basics-t-001-basic-tables-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34477", + "reducers_diff": { + "only_golden": [], + "only_llm": [ + "init()" + ] + }, + "reducers_equal": false, + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:45:58.318770844Z", - "finished_at": "2026-01-23T20:46:19.785596995Z" + "started_at": "2026-01-23T23:54:12.884839011Z", + "finished_at": "2026-01-23T23:54:44.364239410Z" }, "t_002_scheduled_table": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_002_scheduled_table", "lang": "rust", "golden_published": false, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat(std::time::Duration::from_micros(50_000)),\n });\n }\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, scheduled(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat(50_000u64),\n });\n }\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext, _scheduled_id: u64) {\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -99,24 +104,24 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.33\n Compiling find-msvc-tools v0.1.8\n Compiling bitflags v2.10.0\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling thiserror v1.0.69\n Compiling arrayvec v0.7.6\n Compiling bytes v1.11.0\n Compiling humantime v2.3.0\n Compiling convert_case v0.4.0\n Compiling heck v0.4.1\n Compiling keccak v0.1.5\n Compiling zmij v1.0.16\n Compiling second-stack v0.3.5\n Compiling arrayref v0.3.9\n Compiling constant_time_eq v0.4.2\n Compiling spacetimedb-lib v1.11.1\n Compiling bytemuck v1.24.0\n Compiling smallvec v1.15.1\n Compiling itoa v1.0.17\n Compiling getrandom v0.2.17\n Compiling itertools v0.12.1\n Compiling serde_json v1.0.149\n Compiling hex v0.4.3\n Compiling memchr v2.7.6\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling cc v1.2.54\n Compiling rand_core v0.6.4\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling blake3 v1.8.3\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:15:36\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:23:44\n |\n23 | pub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:15\n |\n14 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:15:16\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:17:39\n |\n17 | scheduled_at: ScheduleAt::repeat(std::time::Duration::from_micros(50_000)),\n | ^^^^^^ variant or associated item not found in `ScheduleAt`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:23:8\n |\n 22 | #[reducer]\n | ---------- required by a bound introduced by this call\n 23 | pub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `register_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:746:81\n |\n746 | pub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl Reducer<'a, A>) {\n | ^^^^^^^^^^^^^^ required by this bound in `register_reducer`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:23:8\n |\n22 | #[reducer]\n | ---------- required by a bound introduced by this call\n23 | pub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `invoke_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:45:19\n |\n44 | pub fn invoke_reducer<'a, A: Args<'a>>(\n | -------------- required by a bound in this function\n45 | reducer: impl Reducer<'a, A>,\n | ^^^^^^^^^^^^^^ required by this bound in `invoke_reducer`\n\nSome errors have detailed explanations: E0277, E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0277`.\nerror: could not compile `spacetime-module` (lib) due to 8 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling either v1.15.0\n Compiling find-msvc-tools v0.1.8\n Compiling shlex v1.3.0\n Compiling serde v1.0.228\n Compiling zerocopy v0.8.33\n Compiling bitflags v2.10.0\n Compiling anyhow v1.0.100\n Compiling nohash-hasher v0.2.0\n Compiling thiserror v1.0.69\n Compiling zmij v1.0.16\n Compiling heck v0.4.1\n Compiling arrayvec v0.7.6\n Compiling convert_case v0.4.0\n Compiling humantime v2.3.0\n Compiling keccak v0.1.5\n Compiling bytes v1.11.0\n Compiling arrayref v0.3.9\n Compiling itoa v1.0.17\n Compiling hex v0.4.3\n Compiling serde_json v1.0.149\n Compiling getrandom v0.2.17\n Compiling smallvec v1.15.1\n Compiling itertools v0.12.1\n Compiling cc v1.2.54\n Compiling second-stack v0.3.5\n Compiling constant_time_eq v0.4.2\n Compiling spacetimedb-lib v1.11.1\n Compiling rand_core v0.6.4\n Compiling bytemuck v1.24.0\n Compiling generic-array v0.14.7\n Compiling memchr v2.7.6\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling blake3 v1.8.3\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling sha3 v0.10.8\n Compiling decorum v0.3.1\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected `at`\n --> src/lib.rs:4:38\n |\n4 | #[table(name = tick_timer, scheduled(reducer = tick, column = scheduled_at))]\n | ^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:15:36\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:15\n |\n14 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:15:16\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:17:39\n |\n17 | scheduled_at: ScheduleAt::repeat(50_000u64),\n | ^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0422, E0599.\nFor more information about an error, try `rustc --explain E0422`.\nerror: could not compile `spacetime-module` (lib) due to 5 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:45:58.319658080Z", - "finished_at": "2026-01-23T20:46:56.319486952Z" + "started_at": "2026-01-23T23:54:12.885646580Z", + "finished_at": "2026-01-23T23:55:20.697821642Z" }, "t_003_struct_in_table": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_003_struct_in_table", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, - "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table, UniqueColumn, SpacetimeType};\n\n#[derive(SpacetimeType, Clone)]\npub struct Position {\n pub x: i32,\n pub y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n pub id: i32,\n pub pos: Position,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(_ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn insert_entity(ctx: &ReducerContext, id: i32, x: i32, y: i32) {\n ctx.db.entity().insert(Entity { id, pos: Position { x, y } });\n}\n\n#[spacetimedb::reducer]\npub fn set_pos(ctx: &ReducerContext, id: i32, x: i32, y: i32) {\n if let Some(mut e) = ctx.db.entity().id().find(id) {\n e.pos.x = x;\n e.pos.y = y;\n ctx.db.entity().id().update(e);\n }\n}", + "passed_tests": 1, + "llm_output": "use spacetimedb::{table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Position {\n pub x: i32,\n pub y: i32,\n}\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n pub id: i32,\n pub pos: Position,\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-003-struct-in-table-golden", @@ -125,39 +130,32 @@ "work_dir_llm": "target/llm-runs/basics/t_003_struct_in_table/rust/server/gpt-5/llm", "scorer_details": { "schema_parity": { - "pass": false, - "partial": 0.0, + "pass": true, + "partial": 1.0, "notes": { "golden_db": "basics-t-003-struct-in-table-golden", "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", - "reducers_diff": { - "only_golden": [], - "only_llm": [ - "init()", - "insert_entity()", - "set_pos()" - ] - }, - "reducers_equal": false, - "server": "http://127.0.0.1:34477", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:45:58.319022832Z", - "finished_at": "2026-01-23T20:46:29.838664307Z" + "started_at": "2026-01-23T23:54:12.885049901Z", + "finished_at": "2026-01-23T23:54:44.022438380Z" }, "t_004_insert": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_004_insert", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 2, "passed_tests": 1, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().insert(User { id, name, age, active });\n}", + "llm_output": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().insert(User { id, name, age, active });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-004-insert-golden", @@ -173,7 +171,7 @@ "llm_db": "basics-t-004-insert-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34477", + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } @@ -182,24 +180,24 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `user`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c200e9fa53e3a843d1820d588423033ed683af3525a68a4dc3f8578dabd3b437/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `user`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37093/v1/database/c2002cd40e701250226f73cd286f21d0fabecee66409ea7072d8547d122d732f/sql)\n", "phase": "sql_golden" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:45:58.319345443Z", - "finished_at": "2026-01-23T20:46:19.313225077Z" + "started_at": "2026-01-23T23:54:12.885355239Z", + "finished_at": "2026-01-23T23:54:36.594727744Z" }, "t_005_update": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_005_update", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n let new_row = User { id, name, age, active };\n if ctx.db.user().id().find(id).is_some() {\n ctx.db.user().id().update(new_row);\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n let user_tbl = ctx.db.user();\n if user_tbl.id().find(id).is_some() {\n let row = User { id, name, age, active };\n let _ = user_tbl.id().update(row);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -223,14 +221,7 @@ "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "query": "SELECT id, name, age, active FROM user WHERE id=1", "reducer": "update_user", - "server": "http://127.0.0.1:34477" - } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" + "server": "http://127.0.0.1:37093" } }, "schema_parity": { @@ -241,25 +232,32 @@ "llm_db": "basics-t-005-update-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34477", + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } + }, + "seed_users_row": { + "pass": true, + "partial": 1.0, + "notes": { + "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" + } } }, "vendor": "openai", - "started_at": "2026-01-23T20:46:19.313268675Z", - "finished_at": "2026-01-23T20:46:50.997884926Z" + "started_at": "2026-01-23T23:54:29.927557906Z", + "finished_at": "2026-01-23T23:55:05.452075958Z" }, "t_006_delete": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_006_delete", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.user().id().delete(&id);\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn delete_user(ctx: &ReducerContext, id: i32) {\n ctx.db.user().id().delete(&id);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-006-delete-golden", @@ -267,6 +265,15 @@ "work_dir_golden": "target/llm-runs/basics/t_006_delete/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_006_delete/rust/server/gpt-5/llm", "scorer_details": { + "delete_user_count_zero": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" + } + }, "seed_users_row": { "pass": true, "partial": 1.0, @@ -282,34 +289,25 @@ "llm_db": "basics-t-006-delete-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34477", + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } - }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" - } } }, "vendor": "openai", - "started_at": "2026-01-23T20:46:21.669348863Z", - "finished_at": "2026-01-23T20:46:43.139558965Z" + "started_at": "2026-01-23T23:54:43.492030285Z", + "finished_at": "2026-01-23T23:55:25.742698140Z" }, "t_007_crud": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_007_crud", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n let users = ctx.db.user();\n\n users.insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n users.insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n\n let updated = User { id: 1, name: \"Alice2\".to_string(), age: 31, active: false };\n let _ = users.id().update(updated);\n\n let id_to_delete = 2;\n users.id().delete(&id_to_delete);\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n let mut u1 = ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n\n let _u2 = ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n\n u1.name = \"Alice2\".to_string();\n u1.age = 31;\n u1.active = false;\n let _u1_updated = ctx.db.user().id().update(u1);\n\n let id2 = 2i32;\n ctx.db.user().id().delete(&id2);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -317,23 +315,23 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm", "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" - } - }, "crud_row_id1_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `user`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c200fecea2f4fd1e842ce9bf311fc7af775ed2f1c09d17c59d34a450538b65ed/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `user`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37093/v1/database/c200fd84162598d608587e2986747375700c8dd4786931b3439125fcb8d48289/sql)\n", "phase": "sql_golden" } }, + "crud_total_count_one": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -342,34 +340,34 @@ "llm_db": "basics-t-007-crud-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34477", + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } }, - "crud_total_count_one": { + "crud_row_id2_deleted": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:45:58.319932266Z", - "finished_at": "2026-01-23T20:46:41.862776693Z" + "started_at": "2026-01-23T23:54:12.885914392Z", + "finished_at": "2026-01-23T23:54:43.758962254Z" }, "t_008_index_lookup": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_008_index_lookup", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n ctx.db.result().id().delete(&u.id);\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name });\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(user) = ctx.db.user().id().find(id) {\n ctx.db.result().id().delete(&id);\n ctx.db.result().insert(ResultRow { id: user.id, name: user.name });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -385,7 +383,7 @@ "llm_db": "basics-t-008-index-lookup-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34477", + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } @@ -410,23 +408,23 @@ "llm_out": "id | name ----+--------- 1 | \"Alice\"", "query": "SELECT id, name FROM result WHERE id=1", "reducer": "lookup_user_name", - "server": "http://127.0.0.1:34477" + "server": "http://127.0.0.1:37093" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:46:20.508322641Z", - "finished_at": "2026-01-23T20:46:56.770679153Z" + "started_at": "2026-01-23T23:54:43.389799379Z", + "finished_at": "2026-01-23T23:55:22.299128028Z" }, "t_009_init": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_009_init", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n let users = ctx.db.user();\n users.insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n users.insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -434,15 +432,6 @@ "work_dir_golden": "target/llm-runs/basics/t_009_init/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_009_init/rust/server/gpt-5/llm", "scorer_details": { - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -451,36 +440,45 @@ "llm_db": "basics-t-009-init-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34477", + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } }, - "init_seed_bob": { + "init_total_two": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM user" + } + }, + "init_seed_alice": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" } }, - "init_total_two": { + "init_seed_bob": { "pass": true, "partial": 1.0, "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:46:19.785638677Z", - "finished_at": "2026-01-23T20:46:42.338746730Z" + "started_at": "2026-01-23T23:54:36.594836950Z", + "finished_at": "2026-01-23T23:55:03.406964508Z" }, "t_010_connect": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_010_connect", "lang": "rust", "golden_published": true, @@ -503,25 +501,25 @@ "llm_db": "basics-t-010-connect-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34477", + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:45:58.319798861Z", - "finished_at": "2026-01-23T20:46:24.449136396Z" + "started_at": "2026-01-23T23:54:12.885779579Z", + "finished_at": "2026-01-23T23:54:43.492006133Z" }, "t_011_helper_function": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_011_helper_function", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 2, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n let sum = add(a, b);\n ctx.db.result().insert(ResultRow { id, sum });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[reducer]\nfn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n ctx.db.result().insert(ResultRow { id, sum: add(a, b) });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-011-helper-function-golden", @@ -529,50 +527,50 @@ "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/rust/server/gpt-5/llm", "scorer_details": { - "helper_func_sum_abs": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" + "golden_db": "basics-t-011-helper-function-golden", + "llm_db": "basics-t-011-helper-function-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:37093", + "tables_diff": null, + "tables_equal": true } }, "helper_func_sum_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `result`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c200a76ada80a129ab1d98b1132742dd63568f85071327226e59b33240f90ed8/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `result`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37093/v1/database/c20077d8eb0a0abaac0c611ea4342c7aea3d0dd1ab6ac233f20ba9d0b1635306/sql)\n", "phase": "sql_golden" } }, - "schema_parity": { + "helper_func_sum_abs": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "basics-t-011-helper-function-golden", - "llm_db": "basics-t-011-helper-function-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34477", - "tables_diff": null, - "tables_equal": true + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:45:58.319183557Z", - "finished_at": "2026-01-23T20:46:20.508270684Z" + "started_at": "2026-01-23T23:54:12.885202257Z", + "finished_at": "2026-01-23T23:54:43.389769168Z" }, "t_012_spacetime_product_type": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_012_spacetime_product_type", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\nstruct Score {\n left: i32,\n right: i32,\n}\n\n#[table(name = result)]\nstruct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[reducer]\nfn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow { id, value: Score { left, right } });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -580,17 +578,22 @@ "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "product_type_row_parity": { "pass": true, "partial": 1.0, "notes": { + "args": [ + 1, + 2, + 3 + ], "golden_db": "schema-t-012-spacetime-product-type-golden", + "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34477", - "tables_diff": null, - "tables_equal": true + "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", + "query": "SELECT id, value FROM result WHERE id=1", + "reducer": "set_score", + "server": "http://127.0.0.1:37093" } }, "product_type_row_count": { @@ -602,38 +605,33 @@ "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" } }, - "product_type_row_parity": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "args": [ - 1, - 2, - 3 - ], "golden_db": "schema-t-012-spacetime-product-type-golden", - "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", - "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_score", - "server": "http://127.0.0.1:34477" + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:37093", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:46:50.997932201Z", - "finished_at": "2026-01-23T20:47:18.590168533Z" + "started_at": "2026-01-23T23:55:19.787335153Z", + "finished_at": "2026-01-23T23:55:43.342662434Z" }, "t_013_spacetime_sum_type": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_013_spacetime_sum_type", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType)]\nstruct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\nenum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = result)]\nstruct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[spacetimedb::reducer]\nfn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().id().delete(&id);\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -641,6 +639,15 @@ "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/gpt-5/llm", "scorer_details": { + "sum_type_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -649,20 +656,11 @@ "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34477", + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } }, - "sum_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" - } - }, "sum_type_row_parity": { "pass": true, "partial": 1.0, @@ -677,23 +675,23 @@ "llm_out": "id | value ----+--------------- 1 | (Circle = 10)", "query": "SELECT id, value FROM result WHERE id=1", "reducer": "set_circle", - "server": "http://127.0.0.1:34477" + "server": "http://127.0.0.1:37093" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:46:56.770721316Z", - "finished_at": "2026-01-23T20:47:22.846786446Z" + "started_at": "2026-01-23T23:55:20.697841019Z", + "finished_at": "2026-01-23T23:55:58.028620416Z" }, "t_014_elementary_columns": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_014_elementary_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3000000000i64,\n price: 1.5f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[reducer]\nfn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-014-elementary-columns-golden", @@ -712,16 +710,7 @@ "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", "reducer": "seed", - "server": "http://127.0.0.1:34477" - } - }, - "elementary_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" + "server": "http://127.0.0.1:37093" } }, "schema_parity": { @@ -732,25 +721,34 @@ "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34477", + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } + }, + "elementary_columns_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" + } } }, "vendor": "openai", - "started_at": "2026-01-23T20:46:56.319506066Z", - "finished_at": "2026-01-23T20:47:21.058802558Z" + "started_at": "2026-01-23T23:55:20.558344928Z", + "finished_at": "2026-01-23T23:55:44.434182296Z" }, "t_015_product_type_columns": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_015_product_type_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".to_string(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".to_string(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let profile = ctx.db.profile();\n\n for row in profile.iter() {\n profile.id().delete(&row.id);\n }\n\n profile.insert(Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".to_string(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".to_string(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -766,11 +764,20 @@ "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34477", + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } }, + "product_type_columns_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" + } + }, "product_type_columns_row_parity": { "pass": true, "partial": 1.0, @@ -782,32 +789,23 @@ "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", "query": "SELECT id, home, work, pos FROM profile WHERE id=1", "reducer": "seed", - "server": "http://127.0.0.1:34477" - } - }, - "product_type_columns_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" + "server": "http://127.0.0.1:37093" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:46:56.770961297Z", - "finished_at": "2026-01-23T20:47:32.576819834Z" + "started_at": "2026-01-23T23:55:22.299174613Z", + "finished_at": "2026-01-23T23:55:51.516097076Z" }, "t_016_sum_type_columns": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_016_sum_type_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let a = Shape::Circle(10);\n let b = Shape::Rectangle(Rect { width: 4, height: 6 });\n ctx.db.drawing().insert(Drawing { id: 1, a, b });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -815,24 +813,11 @@ "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-016-sum-type-columns-golden", - "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:34477", - "tables_diff": null, - "tables_equal": true - } - }, "sum_type_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c2002a23ef16d310345e1d4ce4b5d48569b614371f4967976a5819022bbc2662/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37093/v1/database/c200baf6e05e127bb6c08d0dadc067cc47eff464f996cc516d17c02565c76231/sql)\n", "phase": "sql" } }, @@ -840,24 +825,37 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c20040bc998fdc9ab3d5a21e438444dc0653e63e1691252800cd646cfc6722b3/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37093/v1/database/c20000f2f719d2473685f272970a6ecfa83b00dc746a1a773f4f8901a0e09aec/sql)\n", "phase": "sql_golden" } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "schema-t-016-sum-type-columns-golden", + "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:37093", + "tables_diff": null, + "tables_equal": true + } } }, "vendor": "openai", - "started_at": "2026-01-23T20:47:00.920013549Z", - "finished_at": "2026-01-23T20:47:31.508612717Z" + "started_at": "2026-01-23T23:55:24.734590745Z", + "finished_at": "2026-01-23T23:55:57.654789420Z" }, "t_017_scheduled_columns": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_017_scheduled_columns", "lang": "rust", "golden_published": false, "model_name": "GPT-5", "total_tests": 2, "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n }\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n let tbl = ctx.db.tick_timer();\n if tbl.count() == 0 {\n tbl.insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::RepeatMicros(50_000),\n });\n }\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -869,24 +867,24 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling zerocopy v0.8.33\n Compiling shlex v1.3.0\n Compiling find-msvc-tools v0.1.8\n Compiling bitflags v2.10.0\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling thiserror v1.0.69\n Compiling bytes v1.11.0\n Compiling convert_case v0.4.0\n Compiling keccak v0.1.5\n Compiling arrayvec v0.7.6\n Compiling heck v0.4.1\n Compiling zmij v1.0.16\n Compiling humantime v2.3.0\n Compiling second-stack v0.3.5\n Compiling bytemuck v1.24.0\n Compiling itoa v1.0.17\n Compiling arrayref v0.3.9\n Compiling getrandom v0.2.17\n Compiling constant_time_eq v0.4.2\n Compiling hex v0.4.3\n Compiling itertools v0.12.1\n Compiling spacetimedb-lib v1.11.1\n Compiling serde_json v1.0.149\n Compiling smallvec v1.15.1\n Compiling rand_core v0.6.4\n Compiling memchr v2.7.6\n Compiling generic-array v0.14.7\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling num-traits v0.2.19\n Compiling cc v1.2.54\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling blake3 v1.8.3\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:15:36\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:23:42\n |\n23 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {}\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:15\n |\n14 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:15:16\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:17:39\n |\n17 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:23:8\n |\n 22 | #[reducer]\n | ---------- required by a bound introduced by this call\n 23 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {}\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `register_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:746:81\n |\n746 | pub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl Reducer<'a, A>) {\n | ^^^^^^^^^^^^^^ required by this bound in `register_reducer`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:23:8\n |\n22 | #[reducer]\n | ---------- required by a bound introduced by this call\n23 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {}\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `invoke_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:45:19\n |\n44 | pub fn invoke_reducer<'a, A: Args<'a>>(\n | -------------- required by a bound in this function\n45 | reducer: impl Reducer<'a, A>,\n | ^^^^^^^^^^^^^^ required by this bound in `invoke_reducer`\n\nSome errors have detailed explanations: E0277, E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0277`.\nerror: could not compile `spacetime-module` (lib) due to 8 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling either v1.15.0\n Compiling zerocopy v0.8.33\n Compiling shlex v1.3.0\n Compiling serde v1.0.228\n Compiling find-msvc-tools v0.1.8\n Compiling nohash-hasher v0.2.0\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling anyhow v1.0.100\n Compiling zmij v1.0.16\n Compiling heck v0.4.1\n Compiling convert_case v0.4.0\n Compiling humantime v2.3.0\n Compiling arrayvec v0.7.6\n Compiling keccak v0.1.5\n Compiling bytes v1.11.0\n Compiling constant_time_eq v0.4.2\n Compiling bytemuck v1.24.0\n Compiling second-stack v0.3.5\n Compiling itoa v1.0.17\n Compiling getrandom v0.2.17\n Compiling serde_json v1.0.149\n Compiling hex v0.4.3\n Compiling rand_core v0.6.4\n Compiling arrayref v0.3.9\n Compiling smallvec v1.15.1\n Compiling itertools v0.12.1\n Compiling spacetimedb-lib v1.11.1\n Compiling memchr v2.7.6\n Compiling generic-array v0.14.7\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling num-traits v0.2.19\n Compiling cc v1.2.54\n Compiling syn v2.0.114\n Compiling http v1.4.0\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling blake3 v1.8.3\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:16:20\n |\n16 | tbl.insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:24:42\n |\n24 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:22\n |\n14 | let tbl = ctx.db.tick_timer();\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `RepeatMicros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:18:39\n |\n18 | scheduled_at: ScheduleAt::RepeatMicros(50_000),\n | ^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:24:8\n |\n 23 | #[reducer]\n | ---------- required by a bound introduced by this call\n 24 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `register_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:746:81\n |\n746 | pub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl Reducer<'a, A>) {\n | ^^^^^^^^^^^^^^ required by this bound in `register_reducer`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:24:8\n |\n23 | #[reducer]\n | ---------- required by a bound introduced by this call\n24 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `invoke_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:45:19\n |\n44 | pub fn invoke_reducer<'a, A: Args<'a>>(\n | -------------- required by a bound in this function\n45 | reducer: impl Reducer<'a, A>,\n | ^^^^^^^^^^^^^^ required by this bound in `invoke_reducer`\n\nSome errors have detailed explanations: E0277, E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0277`.\nerror: could not compile `spacetime-module` (lib) due to 7 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:46:43.139606842Z", - "finished_at": "2026-01-23T20:47:30.076456163Z" + "started_at": "2026-01-23T23:55:05.452109524Z", + "finished_at": "2026-01-23T23:56:06.104136012Z" }, "t_018_constraints": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_018_constraints", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let tbl = ctx.db.account();\n\n for row in tbl.iter() {\n tbl.id().delete(&row.id);\n }\n\n tbl.insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n tbl.insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let ids: Vec = ctx.db.account().iter().map(|r| r.id).collect();\n for id in ids {\n ctx.db.account().id().delete(&id);\n }\n ctx.db\n .account()\n .insert(Account { id: 1, email: \"a@example.com\".to_string(), name: \"Alice\".to_string() });\n ctx.db\n .account()\n .insert(Account { id: 2, email: \"b@example.com\".to_string(), name: \"Bob\".to_string() });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-018-constraints-golden", @@ -894,6 +892,15 @@ "work_dir_golden": "target/llm-runs/schema/t_018_constraints/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_018_constraints/rust/server/gpt-5/llm", "scorer_details": { + "constraints_seed_two_rows": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -902,20 +909,11 @@ "llm_db": "schema-t-018-constraints-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34477", + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" - } - }, "constraints_row_parity_after_seed": { "pass": true, "partial": 1.0, @@ -927,23 +925,23 @@ "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", "query": "SELECT id, email, name FROM account WHERE id=1", "reducer": "seed", - "server": "http://127.0.0.1:34477" + "server": "http://127.0.0.1:37093" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:46:41.862819773Z", - "finished_at": "2026-01-23T20:47:07.104266828Z" + "started_at": "2026-01-23T23:54:44.364263416Z", + "finished_at": "2026-01-23T23:55:20.558298479Z" }, "t_019_many_to_many": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_019_many_to_many", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 5, "passed_tests": 5, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[reducer]\nfn seed(ctx: &ReducerContext) {\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n for m in ctx.db.membership().iter() {\n ctx.db.membership().id().delete(&m.id);\n }\n for u in ctx.db.user().iter() {\n ctx.db.user().user_id().delete(&u.user_id);\n }\n for g in ctx.db.group().iter() {\n ctx.db.group().group_id().delete(&g.group_id);\n }\n\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-019-many-to-many-golden", @@ -951,22 +949,22 @@ "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/rust/server/gpt-5/llm", "scorer_details": { - "m2m_has_2_20": { + "m2m_has_1_10": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" } }, - "m2m_has_1_10": { + "memberships_three_rows": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM membership" } }, "m2m_has_1_20": { @@ -986,34 +984,34 @@ "llm_db": "schema-t-019-many-to-many-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34477", + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } }, - "memberships_three_rows": { + "m2m_has_2_20": { "pass": true, "partial": 1.0, "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:46:42.338778389Z", - "finished_at": "2026-01-23T20:47:21.898328928Z" + "started_at": "2026-01-23T23:55:03.407007070Z", + "finished_at": "2026-01-23T23:55:43.880918665Z" }, "t_020_ecs": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_020_ecs", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 5, "passed_tests": 1, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, UniqueColumn};\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let _ = ctx.db.entity().try_insert(Entity { id: 1 });\n let _ = ctx.db.entity().try_insert(Entity { id: 2 });\n\n let _ = ctx.db.position().try_insert(Position { entity_id: 1, x: 0, y: 0 });\n let _ = ctx.db.position().try_insert(Position { entity_id: 2, x: 10, y: 0 });\n\n let _ = ctx.db.velocity().try_insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n let _ = ctx.db.velocity().try_insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n}\n\n#[reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n let next = NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n };\n if ctx.db.next_position().entity_id().find(pos.entity_id).is_some() {\n let _ = ctx.db.next_position().entity_id().update(next);\n } else {\n let _ = ctx.db.next_position().insert(next);\n }\n }\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.entity().insert(Entity { id: 1 });\n ctx.db.position().insert(Position { entity_id: 1, x: 0, y: 0 });\n ctx.db.velocity().insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n\n ctx.db.entity().insert(Entity { id: 2 });\n ctx.db.position().insert(Position { entity_id: 2, x: 10, y: 0 });\n ctx.db.velocity().insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n}\n\n#[reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n\n if let Some(mut existing) = ctx.db.next_position().entity_id().find(pos.entity_id) {\n existing.x = nx;\n existing.y = ny;\n let _ = ctx.db.next_position().entity_id().update(existing);\n } else {\n let _ = ctx.db.next_position().insert(NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n });\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -1021,11 +1019,19 @@ "work_dir_golden": "target/llm-runs/schema/t_020_ecs/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_020_ecs/rust/server/gpt-5/llm", "scorer_details": { - "ecs_seed_positions_count": { + "ecs_next_pos_entity1": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37093/v1/database/c200b735483077302c2220af9b35c9c621fc59499f0c0c65b07c0fa34aad4735/sql)\n", + "phase": "sql" + } + }, + "ecs_next_pos_entity2": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c2002015adb1317fe3abcb13d934628242ad331957f4caa79e06dee69a06a590/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37093/v1/database/c200b735483077302c2220af9b35c9c621fc59499f0c0c65b07c0fa34aad4735/sql)\n", "phase": "sql" } }, @@ -1037,49 +1043,41 @@ "llm_db": "schema-t-020-ecs-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34477", + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } }, - "ecs_step_next_positions_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c2002015adb1317fe3abcb13d934628242ad331957f4caa79e06dee69a06a590/sql)\n", - "phase": "sql" - } - }, - "ecs_next_pos_entity1": { + "ecs_seed_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c2002015adb1317fe3abcb13d934628242ad331957f4caa79e06dee69a06a590/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37093/v1/database/c200b735483077302c2220af9b35c9c621fc59499f0c0c65b07c0fa34aad4735/sql)\n", "phase": "sql" } }, - "ecs_next_pos_entity2": { + "ecs_step_next_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:34477/v1/database/c2002015adb1317fe3abcb13d934628242ad331957f4caa79e06dee69a06a590/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37093/v1/database/c200b735483077302c2220af9b35c9c621fc59499f0c0c65b07c0fa34aad4735/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:46:24.449168444Z", - "finished_at": "2026-01-23T20:47:00.919947661Z" + "started_at": "2026-01-23T23:54:43.759002021Z", + "finished_at": "2026-01-23T23:55:24.734536804Z" }, "t_021_multi_column_index": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_021_multi_column_index", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[reducer]\nfn seed(ctx: &ReducerContext) {\n let table = ctx.db.log();\n let _ = table.try_insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n let _ = table.try_insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n let _ = table.try_insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -1087,6 +1085,15 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/rust/server/gpt-5/llm", "scorer_details": { + "mcindex_seed_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM log" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -1095,7 +1102,7 @@ "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:34477", + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } @@ -1117,20 +1124,11 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" } - }, - "mcindex_seed_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } } }, "vendor": "openai", - "started_at": "2026-01-23T20:46:29.838705690Z", - "finished_at": "2026-01-23T20:46:56.594909460Z" + "started_at": "2026-01-23T23:54:44.022466387Z", + "finished_at": "2026-01-23T23:55:19.787278738Z" } } } @@ -1138,14 +1136,14 @@ }, { "mode": "docs", - "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "models": [ { "name": "GPT-5", "route_api_model": "gpt-5", "tasks": { "t_000_empty_reducers": { - "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_000_empty_reducers", "lang": "rust", "golden_published": true, @@ -1170,18 +1168,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T20:47:32.846152158Z", - "finished_at": "2026-01-23T20:48:46.489399789Z" + "started_at": "2026-01-23T23:56:06.352823477Z", + "finished_at": "2026-01-23T23:57:14.696827098Z" }, "t_001_basic_tables": { - "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_001_basic_tables", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", + "llm_output": "use spacetimedb::Table;\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-001-basic-tables-golden", @@ -1199,47 +1197,47 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T20:47:32.833609179Z", - "finished_at": "2026-01-23T20:48:40.565099319Z" + "started_at": "2026-01-23T23:56:06.340775944Z", + "finished_at": "2026-01-23T23:57:08.995142469Z" }, "t_002_scheduled_table": { - "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_002_scheduled_table", "lang": "rust", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use std::time::Duration;\nuse spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n });\n}", + "llm_output": null, "category": "basics", "route_api_model": "gpt-5", - "golden_db": "basics-t-002-scheduled-table-golden", - "llm_db": "basics-t-002-scheduled-table-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_002_scheduled_table/rust/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm", + "golden_db": null, + "llm_db": null, + "work_dir_golden": null, + "work_dir_llm": null, "scorer_details": { - "schema_parity": { + "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-002-scheduled-table-golden`.\n", - "phase": "describe_golden" + "error": "LLM call timed out", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:47:32.849575285Z", - "finished_at": "2026-01-23T20:48:55.497529150Z" + "started_at": "2026-01-23T23:58:25.923588482Z", + "finished_at": "2026-01-23T23:58:25.923588482Z" }, "t_003_struct_in_table": { - "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_003_struct_in_table", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::SpacetimeType;\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}", + "llm_output": "use spacetimedb::SpacetimeType;\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n pub x: i32,\n pub y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n pub id: i32,\n pub pos: Position,\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-003-struct-in-table-golden", @@ -1257,11 +1255,11 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T20:47:32.836252652Z", - "finished_at": "2026-01-23T20:48:05.833400632Z" + "started_at": "2026-01-23T23:56:06.343277423Z", + "finished_at": "2026-01-23T23:56:45.192100606Z" }, "t_004_insert": { - "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_004_insert", "lang": "rust", "golden_published": true, @@ -1276,36 +1274,36 @@ "work_dir_golden": "target/llm-runs/basics/t_004_insert/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_004_insert/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "data_parity_insert_user": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-004-insert-golden`.\n", - "phase": "describe_golden" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-004-insert-golden`.\n", + "phase": "call_reducer_golden" } }, - "data_parity_insert_user": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-004-insert-golden`.\n", - "phase": "call_reducer_golden" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-004-insert-golden`.\n", + "phase": "describe_golden" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:47:32.842787873Z", - "finished_at": "2026-01-23T20:48:29.038416446Z" + "started_at": "2026-01-23T23:56:06.349640021Z", + "finished_at": "2026-01-23T23:56:49.150223443Z" }, "t_005_update": { - "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_005_update", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().id().update(User { id, name, age, active });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user, public)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) -> Result<(), String> {\n if ctx.db.user().id().find(id).is_some() {\n ctx.db.user().id().update(User { id, name, age, active });\n Ok(())\n } else {\n Err(\"User not found\".to_string())\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -1313,6 +1311,14 @@ "work_dir_golden": "target/llm-runs/basics/t_005_update/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_005_update/rust/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", + "phase": "describe_golden" + } + }, "data_parity_update_user": { "pass": false, "partial": 0.0, @@ -1329,22 +1335,14 @@ "phase": "sql_golden", "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-005-update-golden`.\n", - "phase": "describe_golden" - } } }, "vendor": "openai", - "started_at": "2026-01-23T20:48:05.833493117Z", - "finished_at": "2026-01-23T20:48:31.203040278Z" + "started_at": "2026-01-23T23:56:36.268165981Z", + "finished_at": "2026-01-23T23:57:06.114000259Z" }, "t_006_delete": { - "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_006_delete", "lang": "rust", "golden_published": true, @@ -1387,73 +1385,47 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T20:48:31.203078568Z", - "finished_at": "2026-01-23T20:48:50.906207073Z" + "started_at": "2026-01-23T23:57:06.114026509Z", + "finished_at": "2026-01-23T23:57:33.036931801Z" }, "t_007_crud": { - "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_007_crud", "lang": "rust", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", - "total_tests": 4, - "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn crud(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".into(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".into(), age: 22, active: false });\n ctx.db.user().id().update(User { id: 1, name: \"Alice2\".into(), age: 31, active: false });\n ctx.db.user().id().delete(2);\n}", + "total_tests": 1, + "passed_tests": 0, + "llm_output": null, "category": "basics", "route_api_model": "gpt-5", - "golden_db": "basics-t-007-crud-golden", - "llm_db": "basics-t-007-crud-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_007_crud/rust/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm", + "golden_db": null, + "llm_db": null, + "work_dir_golden": null, + "work_dir_llm": null, "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", - "phase": "describe_golden" - } - }, - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" - } - }, - "crud_total_count_one": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" - } - }, - "crud_row_id1_parity": { + "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", - "phase": "call_reducer_golden" + "error": "LLM call timed out", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:47:32.856090158Z", - "finished_at": "2026-01-23T20:48:19.458976831Z" + "started_at": "2026-01-23T23:58:25.923597248Z", + "finished_at": "2026-01-23T23:58:25.923597248Z" }, "t_008_index_lookup": { - "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_008_index_lookup", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n ctx.db.result().insert(ResultRow { id: u.id, name: u.name.clone() });\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(user) = ctx.db.user().id().find(id) {\n ctx.db.result().insert(ResultRow { id: user.id, name: user.name });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -1461,15 +1433,6 @@ "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/rust/server/gpt-5/llm", "scorer_details": { - "seed_user_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n", - "phase": "sql_golden", - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, "schema_parity": { "pass": false, "partial": 0.0, @@ -1485,21 +1448,30 @@ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n", "phase": "call_reducer_golden" } + }, + "seed_user_row": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n", + "phase": "sql_golden", + "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" + } } }, "vendor": "openai", - "started_at": "2026-01-23T20:48:29.038498151Z", - "finished_at": "2026-01-23T20:48:56.042485955Z" + "started_at": "2026-01-23T23:56:49.150284606Z", + "finished_at": "2026-01-23T23:57:17.142408609Z" }, "t_009_init": { - "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_009_init", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".into(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".into(), age: 22, active: false });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".into(),\n age: 30,\n active: true,\n });\n ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".into(),\n age: 22,\n active: false,\n });\n Ok(())\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -1507,30 +1479,22 @@ "work_dir_golden": "target/llm-runs/basics/t_009_init/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_009_init/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-009-init-golden`.\n", - "phase": "describe_golden" - } - }, - "init_seed_bob": { + "init_seed_alice": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" } }, - "init_seed_alice": { + "init_seed_bob": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" } }, "init_total_two": { @@ -1541,21 +1505,29 @@ "expected": 2, "sql": "SELECT COUNT(*) AS n FROM user" } + }, + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-009-init-golden`.\n", + "phase": "describe_golden" + } } }, "vendor": "openai", - "started_at": "2026-01-23T20:48:19.459032083Z", - "finished_at": "2026-01-23T20:48:44.457914124Z" + "started_at": "2026-01-23T23:56:45.192125865Z", + "finished_at": "2026-01-23T23:57:10.060687704Z" }, "t_010_connect": { - "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_010_connect", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = event)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event {\n id: 0,\n kind: \"connected\".to_string(),\n });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event {\n id: 0,\n kind: \"disconnected\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = event)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event { id: 0, kind: \"connected\".to_string() });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event { id: 0, kind: \"disconnected\".to_string() });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-010-connect-golden", @@ -1573,18 +1545,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T20:47:32.852844068Z", - "finished_at": "2026-01-23T20:48:51.666864219Z" + "started_at": "2026-01-23T23:56:06.359406815Z", + "finished_at": "2026-01-23T23:56:36.268095898Z" }, "t_011_helper_function": { - "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_011_helper_function", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[spacetimedb::reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n let sum = add(a, b);\n ctx.db.result().insert(ResultRow { id, sum });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[spacetimedb::reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n sum: add(a, b),\n });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-011-helper-function-golden", @@ -1600,37 +1572,37 @@ "phase": "describe_golden" } }, - "helper_func_sum_parity": { + "helper_func_sum_abs": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-011-helper-function-golden`.\n", - "phase": "call_reducer_golden" + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" } }, - "helper_func_sum_abs": { + "helper_func_sum_parity": { "pass": false, "partial": 0.0, "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-011-helper-function-golden`.\n", + "phase": "call_reducer_golden" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:47:32.839497474Z", - "finished_at": "2026-01-23T20:48:51.151329754Z" + "started_at": "2026-01-23T23:56:06.346316813Z", + "finished_at": "2026-01-23T23:57:35.149751175Z" }, "t_012_spacetime_product_type": { - "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_012_spacetime_product_type", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[derive(spacetimedb::SpacetimeType, Clone, Debug)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n let value = Score { left, right };\n ctx.db.result().insert(ResultRow { id, value });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -1638,12 +1610,13 @@ "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/gpt-5/llm", "scorer_details": { - "product_type_row_parity": { + "product_type_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-012-spacetime-product-type-golden`.\n", - "phase": "call_reducer_golden" + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" } }, "schema_parity": { @@ -1654,22 +1627,21 @@ "phase": "describe_golden" } }, - "product_type_row_count": { + "product_type_row_parity": { "pass": false, "partial": 0.0, "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-012-spacetime-product-type-golden`.\n", + "phase": "call_reducer_golden" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:48:51.666947087Z", - "finished_at": "2026-01-23T20:49:14.419221949Z" + "started_at": "2026-01-23T23:57:35.149826204Z", + "finished_at": "2026-01-23T23:57:59.916792151Z" }, "t_013_spacetime_sum_type": { - "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_013_spacetime_sum_type", "lang": "rust", "golden_published": true, @@ -1684,6 +1656,14 @@ "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/gpt-5/llm", "scorer_details": { + "sum_type_row_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-013-spacetime-sum-type-golden`.\n", + "phase": "call_reducer_golden" + } + }, "schema_parity": { "pass": false, "partial": 0.0, @@ -1700,29 +1680,21 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" } - }, - "sum_type_row_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-013-spacetime-sum-type-golden`.\n", - "phase": "call_reducer_golden" - } } }, "vendor": "openai", - "started_at": "2026-01-23T20:48:56.042516286Z", - "finished_at": "2026-01-23T20:49:19.391193690Z" + "started_at": "2026-01-23T23:57:36.363684916Z", + "finished_at": "2026-01-23T23:57:57.614971862Z" }, "t_014_elementary_columns": { - "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_014_elementary_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5,\n ratio: 2.25,\n active: true,\n name: \"Alice\".into(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-014-elementary-columns-golden", @@ -1730,13 +1702,12 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/rust/server/gpt-5/llm", "scorer_details": { - "elementary_columns_row_count": { + "elementary_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", + "phase": "call_reducer_golden" } }, "schema_parity": { @@ -1747,28 +1718,29 @@ "phase": "describe_golden" } }, - "elementary_columns_row_parity": { + "elementary_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", - "phase": "call_reducer_golden" + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:48:55.497603714Z", - "finished_at": "2026-01-23T20:49:21.590560437Z" + "started_at": "2026-01-23T23:57:36.357778606Z", + "finished_at": "2026-01-23T23:57:59.614595614Z" }, "t_015_product_type_columns": { - "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_015_product_type_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.profile().iter() {\n ctx.db.profile().id().delete(&row.id);\n }\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".to_string(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".to_string(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for p in ctx.db.profile().iter() {\n ctx.db.profile().id().delete(&p.id);\n }\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".to_string(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".to_string(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -1776,15 +1748,7 @@ "work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", - "phase": "describe_golden" - } - }, - "product_type_columns_row_parity": { + "product_type_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { @@ -1800,21 +1764,29 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" } + }, + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", + "phase": "describe_golden" + } } }, "vendor": "openai", - "started_at": "2026-01-23T20:49:09.915973191Z", - "finished_at": "2026-01-23T20:49:35.664676341Z" + "started_at": "2026-01-23T23:57:40.591120346Z", + "finished_at": "2026-01-23T23:58:19.356995796Z" }, "t_016_sum_type_columns": { - "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_016_sum_type_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.drawing().iter() {\n ctx.db.drawing().id().delete(row.id);\n }\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.drawing().iter() {\n ctx.db.drawing().id().delete(&row.id);\n }\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -1822,44 +1794,44 @@ "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/gpt-5/llm", "scorer_details": { - "sum_type_columns_row_count": { + "sum_type_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38097/v1/database/c200de67ac503f1e38be9c09680d3df61e49eb3a7d66ef3c6c8c371fb66d4edf/sql)\n", - "phase": "sql" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", + "phase": "call_reducer_golden" } }, - "schema_parity": { + "sum_type_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", - "phase": "describe_golden" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40199/v1/database/c200b6269c628fcac98f15b7068e57501de59c811b92dafc5267eddaca62177b/sql)\n", + "phase": "sql" } }, - "sum_type_columns_row_parity": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", - "phase": "call_reducer_golden" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", + "phase": "describe_golden" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:49:10.455837761Z", - "finished_at": "2026-01-23T20:49:49.430676143Z" + "started_at": "2026-01-23T23:57:48.299178998Z", + "finished_at": "2026-01-23T23:58:25.922826951Z" }, "t_017_scheduled_columns": { - "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_017_scheduled_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 2, "passed_tests": 1, - "llm_output": "use std::time::Duration;\nuse spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, ScheduleAt, Table};\nuse std::time::Duration;\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) -> Result<(), String> {\n Ok(())\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n let interval = ScheduleAt::Interval(Duration::from_micros(50_000).into());\n ctx.db.tick_timer().try_insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: interval,\n })?;\n Ok(())\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -1867,6 +1839,14 @@ "work_dir_golden": "target/llm-runs/schema/t_017_scheduled_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-017-scheduled-columns-golden`.\n", + "phase": "describe_golden" + } + }, "scheduled_seeded_one_row": { "pass": true, "partial": 1.0, @@ -1875,29 +1855,21 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE scheduled_id>=0" } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-017-scheduled-columns-golden`.\n", - "phase": "describe_golden" - } } }, "vendor": "openai", - "started_at": "2026-01-23T20:48:51.151415261Z", - "finished_at": "2026-01-23T20:49:22.096623158Z" + "started_at": "2026-01-23T23:57:33.036962112Z", + "finished_at": "2026-01-23T23:58:11.840818516Z" }, "t_018_constraints": { - "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_018_constraints", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.account().insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n ctx.db.account().insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.account().try_insert(Account {\n id: 1,\n email: \"a@example.com\".into(),\n name: \"Alice\".into(),\n })?;\n ctx.db.account().try_insert(Account {\n id: 2,\n email: \"b@example.com\".into(),\n name: \"Bob\".into(),\n })?;\n Ok(())\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-018-constraints-golden", @@ -1905,14 +1877,6 @@ "work_dir_golden": "target/llm-runs/schema/t_018_constraints/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_018_constraints/rust/server/gpt-5/llm", "scorer_details": { - "constraints_row_parity_after_seed": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-018-constraints-golden`.\n", - "phase": "call_reducer_golden" - } - }, "constraints_seed_two_rows": { "pass": false, "partial": 0.0, @@ -1929,50 +1893,94 @@ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-018-constraints-golden`.\n", "phase": "describe_golden" } + }, + "constraints_row_parity_after_seed": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-018-constraints-golden`.\n", + "phase": "call_reducer_golden" + } } }, "vendor": "openai", - "started_at": "2026-01-23T20:48:46.489485052Z", - "finished_at": "2026-01-23T20:49:14.344881271Z" + "started_at": "2026-01-23T23:57:14.696906982Z", + "finished_at": "2026-01-23T23:57:40.591097329Z" }, "t_019_many_to_many": { - "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_019_many_to_many", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 5, + "passed_tests": 4, + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[spacetimedb::table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[spacetimedb::table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-019-many-to-many-golden", + "llm_db": "schema-t-019-many-to-many-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/rust/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { + "memberships_three_rows": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM membership" + } + }, + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "POST https://api.openai.com/v1/responses -> 502 Bad Gateway: \r\n502 Bad Gateway\r\n\r\n

502 Bad Gateway

\r\n
cloudflare
\r\n\r\n\r\n", - "phase": "build_or_publish" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-019-many-to-many-golden`.\n", + "phase": "describe_golden" + } + }, + "m2m_has_1_10": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" + } + }, + "m2m_has_1_20": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" + } + }, + "m2m_has_2_20": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:49:49.431161113Z", - "finished_at": "2026-01-23T20:49:49.431161113Z" + "started_at": "2026-01-23T23:57:17.142551153Z", + "finished_at": "2026-01-23T23:57:49.896790577Z" }, "t_020_ecs": { - "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_020_ecs", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 5, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n let _ = ctx.db.entity().try_insert(Entity { id: 1 });\n let _ = ctx.db.entity().try_insert(Entity { id: 2 });\n\n let _ = ctx.db.position().try_insert(Position { entity_id: 1, x: 0, y: 0 });\n let _ = ctx.db.velocity().try_insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n\n let _ = ctx.db.position().try_insert(Position { entity_id: 2, x: 10, y: 0 });\n let _ = ctx.db.velocity().try_insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n if let Some(mut existing) = ctx.db.next_position().entity_id().find(pos.entity_id) {\n existing.x = nx;\n existing.y = ny;\n ctx.db.next_position().entity_id().update(existing);\n } else {\n ctx.db.next_position().insert(NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n });\n }\n }\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // entity 1\n let e1 = 1;\n if ctx.db.entity().id().find(e1).is_none() {\n ctx.db.entity().insert(Entity { id: e1 });\n }\n let p1 = Position { entity_id: e1, x: 0, y: 0 };\n if ctx.db.position().entity_id().find(e1).is_some() {\n ctx.db.position().entity_id().update(p1);\n } else {\n ctx.db.position().insert(p1);\n }\n let v1 = Velocity { entity_id: e1, vx: 1, vy: 0 };\n if ctx.db.velocity().entity_id().find(e1).is_some() {\n ctx.db.velocity().entity_id().update(v1);\n } else {\n ctx.db.velocity().insert(v1);\n }\n\n // entity 2\n let e2 = 2;\n if ctx.db.entity().id().find(e2).is_none() {\n ctx.db.entity().insert(Entity { id: e2 });\n }\n let p2 = Position { entity_id: e2, x: 10, y: 0 };\n if ctx.db.position().entity_id().find(e2).is_some() {\n ctx.db.position().entity_id().update(p2);\n } else {\n ctx.db.position().insert(p2);\n }\n let v2 = Velocity { entity_id: e2, vx: -2, vy: 3 };\n if ctx.db.velocity().entity_id().find(e2).is_some() {\n ctx.db.velocity().entity_id().update(v2);\n } else {\n ctx.db.velocity().insert(v2);\n }\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n let np = NextPosition { entity_id: pos.entity_id, x: nx, y: ny };\n if ctx.db.next_position().entity_id().find(pos.entity_id).is_some() {\n ctx.db.next_position().entity_id().update(np);\n } else {\n ctx.db.next_position().insert(np);\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -1980,27 +1988,27 @@ "work_dir_golden": "target/llm-runs/schema/t_020_ecs/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_020_ecs/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "ecs_next_pos_entity1": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-020-ecs-golden`.\n", - "phase": "describe_golden" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40199/v1/database/c200a61076d09b2b7988361f7eb75cbbe85ce3a70a733b7cf52469970ead2ae5/sql)\n", + "phase": "sql" } }, - "ecs_next_pos_entity2": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38097/v1/database/c200af80055dc33059e50f551016932cbf26d8f0f12ac2d6c5d1b5939a8d9aa1/sql)\n", - "phase": "sql" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-020-ecs-golden`.\n", + "phase": "describe_golden" } }, - "ecs_next_pos_entity1": { + "ecs_next_pos_entity2": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38097/v1/database/c200af80055dc33059e50f551016932cbf26d8f0f12ac2d6c5d1b5939a8d9aa1/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40199/v1/database/c200a61076d09b2b7988361f7eb75cbbe85ce3a70a733b7cf52469970ead2ae5/sql)\n", "phase": "sql" } }, @@ -2008,7 +2016,7 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38097/v1/database/c200af80055dc33059e50f551016932cbf26d8f0f12ac2d6c5d1b5939a8d9aa1/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40199/v1/database/c200a61076d09b2b7988361f7eb75cbbe85ce3a70a733b7cf52469970ead2ae5/sql)\n", "phase": "sql" } }, @@ -2016,24 +2024,24 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:38097/v1/database/c200af80055dc33059e50f551016932cbf26d8f0f12ac2d6c5d1b5939a8d9aa1/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40199/v1/database/c200a61076d09b2b7988361f7eb75cbbe85ce3a70a733b7cf52469970ead2ae5/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:48:40.565127071Z", - "finished_at": "2026-01-23T20:49:16.253635323Z" + "started_at": "2026-01-23T23:57:08.995173371Z", + "finished_at": "2026-01-23T23:57:51.538787094Z" }, "t_021_multi_column_index": { - "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_021_multi_column_index", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", + "llm_output": "use spacetimedb::Table;\n\n#[spacetimedb::table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &spacetimedb::ReducerContext) {\n let ids: Vec = ctx.db.log().iter().map(|r| r.id).collect();\n for id in ids {\n ctx.db.log().id().delete(id);\n }\n ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".into() });\n ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".into() });\n ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".into() });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -2041,21 +2049,13 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/rust/server/gpt-5/llm", "scorer_details": { - "mcindex_seed_count": { + "mcindex_lookup_u7_d2": { "pass": true, "partial": 1.0, "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" - } - }, - "schema_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-021-multi-column-index-golden`.\n", - "phase": "describe_golden" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" } }, "mcindex_lookup_u7_d1": { @@ -2067,19 +2067,27 @@ "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" } }, - "mcindex_lookup_u7_d2": { + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-021-multi-column-index-golden`.\n", + "phase": "describe_golden" + } + }, + "mcindex_seed_count": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM log" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:48:44.457949164Z", - "finished_at": "2026-01-23T20:49:10.455801685Z" + "started_at": "2026-01-23T23:57:10.060719477Z", + "finished_at": "2026-01-23T23:57:48.299152677Z" } } } @@ -2270,14 +2278,14 @@ "modes": [ { "mode": "docs", - "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "models": [ { "name": "GPT-5", "route_api_model": "gpt-5", "tasks": { "t_000_empty_reducers": { - "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_000_empty_reducers", "lang": "csharp", "golden_published": true, @@ -2300,18 +2308,18 @@ "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:41165", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:51:58.516007671Z", - "finished_at": "2026-01-23T20:52:19.852644961Z" + "started_at": "2026-01-24T00:00:26.239129005Z", + "finished_at": "2026-01-24T00:01:25.207476766Z" }, "t_001_basic_tables": { - "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_001_basic_tables", "lang": "csharp", "golden_published": true, @@ -2334,52 +2342,47 @@ "llm_db": "basics-t-001-basic-tables-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:41165", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:51:08.718544439Z", - "finished_at": "2026-01-23T20:51:58.515971496Z" + "started_at": "2026-01-23T23:59:46.761379627Z", + "finished_at": "2026-01-24T00:00:40.834487811Z" }, "t_002_scheduled_table": { - "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_002_scheduled_table", "lang": "csharp", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", "total_tests": 1, - "passed_tests": 1, - "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer tickTimer)\n {\n // Scheduled reducer invoked by TickTimer; no-op by specification.\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n if (ctx.Db.TickTimer.Count == 0)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n }\n}", + "passed_tests": 0, + "llm_output": null, "category": "basics", "route_api_model": "gpt-5", - "golden_db": "basics-t-002-scheduled-table-golden", - "llm_db": "basics-t-002-scheduled-table-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_002_scheduled_table/csharp/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_002_scheduled_table/csharp/server/gpt-5/llm", + "golden_db": null, + "llm_db": null, + "work_dir_golden": null, + "work_dir_llm": null, "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, + "publish_error": { + "pass": false, + "partial": 0.0, "notes": { - "golden_db": "basics-t-002-scheduled-table-golden", - "llm_db": "basics-t-002-scheduled-table-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:46871", - "tables_diff": null, - "tables_equal": true + "error": "POST https://api.openai.com/v1/responses -> 500 Internal Server Error: {\n \"error\": {\n \"message\": \"An error occurred while processing your request. You can retry your request, or contact us through our help center at help.openai.com if the error persists. Please include the request ID req_52a83fd28c3442fd8fef03cd91bfc2b9 in your message.\",\n \"type\": \"server_error\",\n \"param\": null,\n \"code\": \"server_error\"\n }\n}", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:52:02.545010135Z", - "finished_at": "2026-01-23T20:52:47.129183717Z" + "started_at": "2026-01-24T00:05:38.729207961Z", + "finished_at": "2026-01-24T00:05:38.729207961Z" }, "t_003_struct_in_table": { - "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_003_struct_in_table", "lang": "csharp", "golden_published": true, @@ -2402,18 +2405,18 @@ "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:41165", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:51:08.721057407Z", - "finished_at": "2026-01-23T20:52:05.996428958Z" + "started_at": "2026-01-23T23:59:46.763860791Z", + "finished_at": "2026-01-24T00:00:48.083440124Z" }, "t_004_insert": { - "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_004_insert", "lang": "csharp", "golden_published": true, @@ -2428,19 +2431,6 @@ "work_dir_golden": "target/llm-runs/basics/t_004_insert/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_004_insert/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-004-insert-golden", - "llm_db": "basics-t-004-insert-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:46871", - "tables_diff": null, - "tables_equal": true - } - }, "data_parity_insert_user": { "pass": true, "partial": 1.0, @@ -2457,23 +2447,36 @@ "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "InsertUser", - "server": "http://127.0.0.1:46871" + "server": "http://127.0.0.1:41165" + } + }, + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-004-insert-golden", + "llm_db": "basics-t-004-insert-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41165", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:51:08.727492764Z", - "finished_at": "2026-01-23T20:52:08.554796558Z" + "started_at": "2026-01-23T23:59:46.769869807Z", + "finished_at": "2026-01-24T00:00:35.226129696Z" }, "t_005_update": { - "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_005_update", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n var updated = new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n };\n\n if (ctx.Db.User.Id.Find(id) != null)\n {\n ctx.Db.User.Id.Update(updated);\n }\n else\n {\n ctx.Db.User.Insert(updated);\n }\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n user.Id = id;\n user.Name = name;\n user.Age = age;\n user.Active = active;\n ctx.Db.User.Id.Update(user);\n }\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -2489,7 +2492,7 @@ "llm_db": "basics-t-005-update-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:41165", "tables_diff": null, "tables_equal": true } @@ -2510,7 +2513,7 @@ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "UpdateUser", - "server": "http://127.0.0.1:46871" + "server": "http://127.0.0.1:41165" } }, "seed_users_row": { @@ -2522,18 +2525,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T20:52:19.852673232Z", - "finished_at": "2026-01-23T20:52:53.389808270Z" + "started_at": "2026-01-24T00:00:48.083499328Z", + "finished_at": "2026-01-24T00:02:11.608478542Z" }, "t_006_delete": { - "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_006_delete", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.User.Id.Delete(id);\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.User.Id.Delete(id);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-006-delete-golden", @@ -2550,13 +2553,6 @@ "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1" } }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -2565,25 +2561,32 @@ "llm_db": "basics-t-006-delete-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:41165", "tables_diff": null, "tables_equal": true } + }, + "seed_users_row": { + "pass": true, + "partial": 1.0, + "notes": { + "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" + } } }, "vendor": "openai", - "started_at": "2026-01-23T20:52:51.107488271Z", - "finished_at": "2026-01-23T20:53:13.731066286Z" + "started_at": "2026-01-24T00:02:06.759067811Z", + "finished_at": "2026-01-24T00:02:35.245051100Z" }, "t_007_crud": { - "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_007_crud", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n ctx.Db.User.Id.Update(new User { Id = 1, Name = \"Alice2\", Age = 31, Active = false });\n ctx.Db.User.Id.Delete(2);\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n if (ctx.Db.User.Id.Find(1) is User u1)\n {\n u1.Name = \"Alice2\";\n u1.Age = 31;\n u1.Active = false;\n ctx.Db.User.Id.Update(u1);\n }\n\n ctx.Db.User.Id.Delete(2);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -2591,6 +2594,15 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/csharp/server/gpt-5/llm", "scorer_details": { + "crud_row_id2_deleted": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2" + } + }, "crud_row_id1_parity": { "pass": true, "partial": 1.0, @@ -2602,7 +2614,7 @@ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "Crud", - "server": "http://127.0.0.1:46871" + "server": "http://127.0.0.1:41165" } }, "crud_total_count_one": { @@ -2614,15 +2626,6 @@ "sql": "SELECT COUNT(*) AS n FROM User" } }, - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -2631,25 +2634,25 @@ "llm_db": "basics-t-007-crud-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:41165", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:52:08.554828637Z", - "finished_at": "2026-01-23T20:52:51.107450441Z" + "started_at": "2026-01-24T00:00:44.726730327Z", + "finished_at": "2026-01-24T00:02:06.759033129Z" }, "t_008_index_lookup": { - "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_008_index_lookup", "lang": "csharp", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 0, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n var user = ctx.Db.User.Id.Find(id);\n if (user != null)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = user.Id,\n Name = user.Name\n });\n }\n }\n}", + "passed_tests": 3, + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial class User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial class Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n var user = ctx.Db.User.Id.Find(id);\n if (user != null)\n {\n ctx.Db.Result.Insert(new Result { Id = user.Id, Name = user.Name });\n }\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -2657,28 +2660,56 @@ "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\n/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm/Lib.cs(32,27): error CS1061: 'Module.User?' does not contain a definition for 'Id' and no accessible extension method 'Id' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm/StdbModule.csproj]\n/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm/Lib.cs(33,29): error CS1061: 'Module.User?' does not contain a definition for 'Name' and no accessible extension method 'Name' accepting a first argument of type 'Module.User?' could be found (are you missing a using directive or an assembly reference?) [/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_008_index_lookup/csharp/server/gpt-5/llm/StdbModule.csproj]\n", - "phase": "build_or_publish" - } - } - }, - "vendor": "openai", - "started_at": "2026-01-23T20:52:47.129209751Z", - "finished_at": "2026-01-23T20:53:08.577581818Z" + "golden_db": "basics-t-008-index-lookup-golden", + "llm_db": "basics-t-008-index-lookup-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41165", + "tables_diff": null, + "tables_equal": true + } + }, + "seed_user_row": { + "pass": true, + "partial": 1.0, + "notes": { + "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" + } + }, + "index_lookup_projection_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [ + 1 + ], + "golden_db": "basics-t-008-index-lookup-golden", + "golden_out": "Id | Name ----+--------- 1 | \"Alice\"", + "llm_db": "basics-t-008-index-lookup-gpt-5-llm", + "llm_out": "Id | Name ----+--------- 1 | \"Alice\"", + "query": "SELECT Id, Name FROM Result WHERE Id=1", + "reducer": "LookupUserName", + "server": "http://127.0.0.1:41165" + } + } + }, + "vendor": "openai", + "started_at": "2026-01-24T00:01:34.491862134Z", + "finished_at": "2026-01-24T00:02:24.983219977Z" }, "t_009_init": { - "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_009_init", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -2686,60 +2717,60 @@ "work_dir_golden": "target/llm-runs/basics/t_009_init/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_009_init/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "init_total_two": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "basics-t-009-init-golden", - "llm_db": "basics-t-009-init-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:46871", - "tables_diff": null, - "tables_equal": true + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM User" } }, - "init_seed_bob": { + "init_seed_alice": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" } }, - "init_total_two": { + "init_seed_bob": { "pass": true, "partial": 1.0, "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM User" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" } }, - "init_seed_alice": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" + "golden_db": "basics-t-009-init-golden", + "llm_db": "basics-t-009-init-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41165", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:52:36.884011764Z", - "finished_at": "2026-01-23T20:53:00.062192214Z" + "started_at": "2026-01-24T00:01:25.207506306Z", + "finished_at": "2026-01-24T00:02:23.314749141Z" }, "t_010_connect": { - "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_010_connect", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Event\", Public = true)]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"disconnected\" });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Event\")]\n public partial struct Event\n {\n [SpacetimeDB.PrimaryKey]\n [SpacetimeDB.AutoInc]\n public int Id;\n public string Kind;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientConnected)]\n public static void ClientConnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"connected\" });\n }\n\n [SpacetimeDB.Reducer(ReducerKind.ClientDisconnected)]\n public static void ClientDisconnected(ReducerContext ctx)\n {\n ctx.Db.Event.Insert(new Event { Id = 0, Kind = \"disconnected\" });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-010-connect-golden", @@ -2755,25 +2786,25 @@ "llm_db": "basics-t-010-connect-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:41165", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:52:05.996450500Z", - "finished_at": "2026-01-23T20:52:36.883985146Z" + "started_at": "2026-01-24T00:00:40.834521245Z", + "finished_at": "2026-01-24T00:01:34.491841870Z" }, "t_011_helper_function": { - "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_011_helper_function", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n public static int Add(int a, int b) => a + b;\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Sum = Add(a, b)\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n public static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Sum = Add(a, b)\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-011-helper-function-golden", @@ -2781,13 +2812,22 @@ "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/csharp/server/gpt-5/llm", "scorer_details": { - "helper_func_sum_abs": { + "helper_func_sum_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1 AND Sum=5" + "args": [ + 1, + 2, + 3 + ], + "golden_db": "basics-t-011-helper-function-golden", + "golden_out": "Id | Sum ----+----- 1 | 5", + "llm_db": "basics-t-011-helper-function-gpt-5-llm", + "llm_out": "Id | Sum ----+----- 1 | 5", + "query": "SELECT Id, Sum FROM Result WHERE Id=1", + "reducer": "ComputeSum", + "server": "http://127.0.0.1:41165" } }, "schema_parity": { @@ -2798,36 +2838,27 @@ "llm_db": "basics-t-011-helper-function-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:41165", "tables_diff": null, "tables_equal": true } }, - "helper_func_sum_parity": { + "helper_func_sum_abs": { "pass": true, "partial": 1.0, "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_db": "basics-t-011-helper-function-golden", - "golden_out": "Id | Sum ----+----- 1 | 5", - "llm_db": "basics-t-011-helper-function-gpt-5-llm", - "llm_out": "Id | Sum ----+----- 1 | 5", - "query": "SELECT Id, Sum FROM Result WHERE Id=1", - "reducer": "ComputeSum", - "server": "http://127.0.0.1:46871" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1 AND Sum=5" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:51:08.724127650Z", - "finished_at": "2026-01-23T20:52:02.544972253Z" + "started_at": "2026-01-23T23:59:46.766827274Z", + "finished_at": "2026-01-24T00:00:26.239091882Z" }, "t_012_spacetime_product_type": { - "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_012_spacetime_product_type", "lang": "csharp", "golden_published": true, @@ -2850,20 +2881,11 @@ "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:41165", "tables_diff": null, "tables_equal": true } }, - "product_type_row_count": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" - } - }, "product_type_row_parity": { "pass": true, "partial": 1.0, @@ -2879,23 +2901,32 @@ "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)", "query": "SELECT Id, Value FROM Result WHERE Id=1", "reducer": "SetScore", - "server": "http://127.0.0.1:46871" + "server": "http://127.0.0.1:41165" + } + }, + "product_type_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:53:40.420411949Z", - "finished_at": "2026-01-23T20:54:07.766554713Z" + "started_at": "2026-01-24T00:03:16.504458119Z", + "finished_at": "2026-01-24T00:03:55.909124187Z" }, "t_013_spacetime_sum_type": { - "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_013_spacetime_sum_type", "lang": "csharp", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 0, - "llm_output": "using SpacetimeDB;\nusing SpacetimeDB.Types;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n var shape = new Shape.Circle(new Circle { Radius = radius });\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Value = shape\n });\n }\n}", + "passed_tests": 3, + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n var shape = new Shape.Circle(new Circle { Radius = radius });\n ctx.Db.Result.Insert(new Result { Id = id, Value = shape });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -2903,28 +2934,59 @@ "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/csharp/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime build (csharp) failed (exit=1)\n--- stderr ---\nError: command [\"dotnet\", \"publish\", \"-c\", \"Release\", \"-v\", \"quiet\"] exited with code 1\n\n--- stdout ---\n/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_013_spacetime_sum_type/csharp/server/gpt-5/llm/Lib.cs(3,19): error CS0234: The type or namespace name 'Types' does not exist in the namespace 'SpacetimeDB' (are you missing an assembly reference?) [/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_013_spacetime_sum_type/csharp/server/gpt-5/llm/StdbModule.csproj]\n", - "phase": "build_or_publish" + "golden_db": "schema-t-013-spacetime-sum-type-golden", + "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41165", + "tables_diff": null, + "tables_equal": true + } + }, + "sum_type_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" + } + }, + "sum_type_row_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [ + 1, + 10 + ], + "golden_db": "schema-t-013-spacetime-sum-type-golden", + "golden_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))", + "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", + "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))", + "query": "SELECT Id, Value FROM Result WHERE Id=1", + "reducer": "SetCircle", + "server": "http://127.0.0.1:41165" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:54:05.268678013Z", - "finished_at": "2026-01-23T20:54:43.846818259Z" + "started_at": "2026-01-24T00:03:41.610651465Z", + "finished_at": "2026-01-24T00:05:18.400719234Z" }, "t_014_elementary_columns": { - "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_014_elementary_columns", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Primitive\", Public = true)]\n public partial struct Primitive\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Count;\n public long Total;\n public float Price;\n public double Ratio;\n public bool Active;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Primitive.Insert(new Primitive\n {\n Id = 1,\n Count = 2,\n Total = 3000000000L,\n Price = 1.5f,\n Ratio = 2.25,\n Active = true,\n Name = \"Alice\"\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Primitive\")]\n public partial struct Primitive\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Count;\n public long Total;\n public float Price;\n public double Ratio;\n public bool Active;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Primitive.Insert(new Primitive\n {\n Id = 1,\n Count = 2,\n Total = 3000000000L,\n Price = 1.5f,\n Ratio = 2.25,\n Active = true,\n Name = \"Alice\"\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-014-elementary-columns-golden", @@ -2932,20 +2994,20 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm", "scorer_details": { - "elementary_columns_row_count": { + "elementary_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c2003883ecec05b675876359540c06439933bae20ff47c4c334ae2cfa397c954/sql)\n", - "phase": "sql" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41165/v1/database/c200abb30b25499293ed5bc7ba50399490b1822a8b306291a5d7e0d29840ec60/sql)\n", + "phase": "sql_golden" } }, - "elementary_columns_row_parity": { + "elementary_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c200ba25315ff7445dc543e2e42a2309cd493cfe1974181f53e50fa06c464134/sql)\n", - "phase": "sql_golden" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41165/v1/database/c2005e15aa9be491e8444b574ff6c04413d8ed2bb5427a2c042dc8cc81fcd58b/sql)\n", + "phase": "sql" } }, "schema_parity": { @@ -2956,25 +3018,25 @@ "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:41165", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:53:43.824629255Z", - "finished_at": "2026-01-23T20:54:14.326210872Z" + "started_at": "2026-01-24T00:03:26.186891573Z", + "finished_at": "2026-01-24T00:04:04.039047837Z" }, "t_015_product_type_columns": { - "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_015_product_type_columns", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Profile\", Public = true)]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var row in ctx.Db.Profile.Iter())\n {\n ctx.Db.Profile.Id.Delete(row.Id);\n }\n\n ctx.Db.Profile.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 }\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Address\n {\n public string Street;\n public int Zip;\n }\n\n [SpacetimeDB.Type]\n public partial struct Position\n {\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Profile\", Public = true)]\n public partial struct Profile\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Address Home;\n public Address Work;\n public Position Pos;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Profile.Insert(new Profile\n {\n Id = 1,\n Home = new Address { Street = \"1 Main\", Zip = 11111 },\n Work = new Address { Street = \"2 Broad\", Zip = 22222 },\n Pos = new Position { X = 7, Y = 9 }\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -2991,47 +3053,47 @@ "sql": "SELECT COUNT(*) AS n FROM Profile WHERE Id=1" } }, - "schema_parity": { + "product_type_columns_row_parity": { "pass": true, "partial": 1.0, "notes": { + "args": [], "golden_db": "schema-t-015-product-type-columns-golden", + "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:46871", - "tables_diff": null, - "tables_equal": true + "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", + "query": "SELECT Id, Home, Work, Pos FROM Profile WHERE Id=1", + "reducer": "Seed", + "server": "http://127.0.0.1:41165" } }, - "product_type_columns_row_parity": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "args": [], "golden_db": "schema-t-015-product-type-columns-golden", - "golden_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", - "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", - "query": "SELECT Id, Home, Work, Pos FROM Profile WHERE Id=1", - "reducer": "Seed", - "server": "http://127.0.0.1:46871" + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41165", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:54:07.846975920Z", - "finished_at": "2026-01-23T20:54:41.128204875Z" + "started_at": "2026-01-24T00:03:55.909160959Z", + "finished_at": "2026-01-24T00:05:10.850801287Z" }, "t_016_sum_type_columns": { - "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_016_sum_type_columns", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Drawing\")]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n var a = new Shape.Circle(new Circle { Radius = 10 });\n var b = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 });\n ctx.Db.Drawing.Insert(new Drawing { Id = 1, A = a, B = b });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Drawing\", Public = true)]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var existing in ctx.Db.Drawing.Iter())\n {\n ctx.Db.Drawing.Id.Delete(existing.Id);\n }\n\n ctx.Db.Drawing.Insert(new Drawing\n {\n Id = 1,\n A = new Shape.Circle(new Circle { Radius = 10 }),\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -3039,6 +3101,14 @@ "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/csharp/server/gpt-5/llm", "scorer_details": { + "sum_type_columns_row_count": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41165/v1/database/c200758d3d7e090a756802600f5abbdfa92e9f80f78bbe9f16875cc52e63db9f/sql)\n", + "phase": "sql" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -3047,7 +3117,7 @@ "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:41165", "tables_diff": null, "tables_equal": true } @@ -3056,32 +3126,24 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c2002b630c1898fdca0d5df967295e9965d354ea89c64878bb9c71421ac210de/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41165/v1/database/c200e49d308cedd0b63e897b1e9231e41c79da83179130463533c2754beabd3c/sql)\n", "phase": "sql_golden" } - }, - "sum_type_columns_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c200255f75652e874ebf03622fd9c2dfb3b65b42f575d4fa5a95cc36e4182f25/sql)\n", - "phase": "sql" - } } }, "vendor": "openai", - "started_at": "2026-01-23T20:54:07.849642740Z", - "finished_at": "2026-01-23T20:54:53.203074499Z" + "started_at": "2026-01-24T00:04:02.368901035Z", + "finished_at": "2026-01-24T00:05:38.729154807Z" }, "t_017_scheduled_columns": { - "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_017_scheduled_columns", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 2, "passed_tests": 1, - "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer row)\n {\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledId = 0,\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromTicks(500_000))\n });\n }\n}", + "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey, SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer _row)\n {\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledId = 0,\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -3097,7 +3159,7 @@ "llm_db": "schema-t-017-scheduled-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:41165", "tables_diff": null, "tables_equal": true } @@ -3106,17 +3168,17 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c200bcc4292b7995634fb91d48b6baa5c9096e7f85c4b73e7db1b4f1f377a348/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41165/v1/database/c20013446ea0691be1033226a60d1a74e75b1615111df0d10aa679ebc52edc7c/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:53:28.130925948Z", - "finished_at": "2026-01-23T20:54:07.846947117Z" + "started_at": "2026-01-24T00:03:05.250970806Z", + "finished_at": "2026-01-24T00:04:02.368868621Z" }, "t_018_constraints": { - "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_018_constraints", "lang": "csharp", "golden_published": true, @@ -3142,16 +3204,7 @@ "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", "query": "SELECT Id, Email, Name FROM Account WHERE Id=1", "reducer": "Seed", - "server": "http://127.0.0.1:46871" - } - }, - "constraints_seed_two_rows": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Account WHERE Id=2" + "server": "http://127.0.0.1:41165" } }, "schema_parity": { @@ -3162,25 +3215,34 @@ "llm_db": "schema-t-018-constraints-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:41165", "tables_diff": null, "tables_equal": true } + }, + "constraints_seed_two_rows": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Account WHERE Id=2" + } } }, "vendor": "openai", - "started_at": "2026-01-23T20:53:08.577602895Z", - "finished_at": "2026-01-23T20:53:40.420363798Z" + "started_at": "2026-01-24T00:02:24.983294531Z", + "finished_at": "2026-01-24T00:03:05.250921900Z" }, "t_019_many_to_many": { - "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_019_many_to_many", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 5, "passed_tests": 5, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"Group\", Public = true)]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"Membership\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { \"UserId\" })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { \"GroupId\" })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var m in ctx.Db.Membership.Iter())\n {\n ctx.Db.Membership.Id.Delete(m.Id);\n }\n foreach (var g in ctx.Db.Group.Iter())\n {\n ctx.Db.Group.GroupId.Delete(g.GroupId);\n }\n foreach (var u in ctx.Db.User.Iter())\n {\n ctx.Db.User.UserId.Delete(u.UserId);\n }\n\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int UserId;\n public string Name;\n }\n\n [SpacetimeDB.Table(Name = \"Group\")]\n public partial struct Group\n {\n [SpacetimeDB.PrimaryKey]\n public int GroupId;\n public string Title;\n }\n\n [SpacetimeDB.Table(Name = \"Membership\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user\", Columns = new[] { \"UserId\" })]\n [SpacetimeDB.Index.BTree(Name = \"by_group\", Columns = new[] { \"GroupId\" })]\n public partial struct Membership\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int GroupId;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var m in ctx.Db.Membership.Iter())\n {\n ctx.Db.Membership.Id.Delete(m.Id);\n }\n foreach (var u in ctx.Db.User.Iter())\n {\n ctx.Db.User.UserId.Delete(u.UserId);\n }\n foreach (var g in ctx.Db.Group.Iter())\n {\n ctx.Db.Group.GroupId.Delete(g.GroupId);\n }\n\n ctx.Db.User.Insert(new User { UserId = 1, Name = \"Alice\" });\n ctx.Db.User.Insert(new User { UserId = 2, Name = \"Bob\" });\n\n ctx.Db.Group.Insert(new Group { GroupId = 10, Title = \"Admin\" });\n ctx.Db.Group.Insert(new Group { GroupId = 20, Title = \"Dev\" });\n\n ctx.Db.Membership.Insert(new Membership { Id = 1, UserId = 1, GroupId = 10 });\n ctx.Db.Membership.Insert(new Membership { Id = 2, UserId = 1, GroupId = 20 });\n ctx.Db.Membership.Insert(new Membership { Id = 3, UserId = 2, GroupId = 20 });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-019-many-to-many-golden", @@ -3188,6 +3250,24 @@ "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm", "scorer_details": { + "m2m_has_1_20": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=20" + } + }, + "m2m_has_1_10": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=10" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -3196,7 +3276,7 @@ "llm_db": "schema-t-019-many-to-many-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:46871", + "server": "http://127.0.0.1:41165", "tables_diff": null, "tables_equal": true } @@ -3218,105 +3298,50 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=2 AND GroupId=20" } - }, - "m2m_has_1_10": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=10" - } - }, - "m2m_has_1_20": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=20" - } } }, "vendor": "openai", - "started_at": "2026-01-23T20:53:13.731095349Z", - "finished_at": "2026-01-23T20:54:05.268633334Z" + "started_at": "2026-01-24T00:02:35.245089256Z", + "finished_at": "2026-01-24T00:03:26.186843126Z" }, "t_020_ecs": { - "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_020_ecs", "lang": "csharp", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", - "total_tests": 5, - "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Entity\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"Position\", Public = true)]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Velocity\", Public = true)]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"NextPosition\", Public = true)]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n // Entity 1\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n\n // Entity 2\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.Position.Iter())\n {\n var vel = ctx.Db.Velocity.EntityId.Find(pos.EntityId);\n if (vel == null)\n {\n continue;\n }\n\n int nextX = pos.X + vel.Value.VX;\n int nextY = pos.Y + vel.Value.VY;\n\n var existing = ctx.Db.NextPosition.EntityId.Find(pos.EntityId);\n if (existing != null)\n {\n var np = existing.Value;\n np.X = nextX;\n np.Y = nextY;\n ctx.Db.NextPosition.EntityId.Update(np);\n }\n else\n {\n ctx.Db.NextPosition.Insert(new NextPosition\n {\n EntityId = pos.EntityId,\n X = nextX,\n Y = nextY\n });\n }\n }\n }\n}", + "total_tests": 1, + "passed_tests": 0, + "llm_output": null, "category": "schema", "route_api_model": "gpt-5", - "golden_db": "schema-t-020-ecs-golden", - "llm_db": "schema-t-020-ecs-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_020_ecs/csharp/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm", + "golden_db": null, + "llm_db": null, + "work_dir_golden": null, + "work_dir_llm": null, "scorer_details": { - "ecs_next_pos_entity1": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c20001cd05b3f1e81ac991a3d37765e210ae358de6cdec43d4918b54b938eee1/sql)\n", - "phase": "sql" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-020-ecs-golden", - "llm_db": "schema-t-020-ecs-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:46871", - "tables_diff": null, - "tables_equal": true - } - }, - "ecs_next_pos_entity2": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c20001cd05b3f1e81ac991a3d37765e210ae358de6cdec43d4918b54b938eee1/sql)\n", - "phase": "sql" - } - }, - "ecs_step_next_positions_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c20001cd05b3f1e81ac991a3d37765e210ae358de6cdec43d4918b54b938eee1/sql)\n", - "phase": "sql" - } - }, - "ecs_seed_positions_count": { + "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:46871/v1/database/c20001cd05b3f1e81ac991a3d37765e210ae358de6cdec43d4918b54b938eee1/sql)\n", - "phase": "sql" + "error": "LLM call timed out", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:52:53.389850738Z", - "finished_at": "2026-01-23T20:53:43.824579798Z" + "started_at": "2026-01-24T00:05:38.729220337Z", + "finished_at": "2026-01-24T00:05:38.729220337Z" }, "t_021_multi_column_index": { - "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_021_multi_column_index", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Log\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = new[] { \"UserId\", \"Day\" })]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Log.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.Log.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.Log.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Log\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = new[] { \"UserId\", \"Day\" })]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Log.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.Log.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.Log.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -3324,50 +3349,50 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/gpt-5/llm", "scorer_details": { - "mcindex_lookup_u7_d1": { + "mcindex_lookup_u7_d2": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=1" + "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=2" } }, - "schema_parity": { + "mcindex_seed_count": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "schema-t-021-multi-column-index-golden", - "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:46871", - "tables_diff": null, - "tables_equal": true + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM Log" } }, - "mcindex_lookup_u7_d2": { + "mcindex_lookup_u7_d1": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=2" + "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=1" } }, - "mcindex_seed_count": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Log" + "golden_db": "schema-t-021-multi-column-index-golden", + "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41165", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:53:00.062219949Z", - "finished_at": "2026-01-23T20:53:28.130894765Z" + "started_at": "2026-01-24T00:02:23.314785957Z", + "finished_at": "2026-01-24T00:03:16.504408864Z" } } } diff --git a/docs/llms/docs-benchmark-summary.json b/docs/llms/docs-benchmark-summary.json index 8fd73afcfd8..aca87b1ed44 100644 --- a/docs/llms/docs-benchmark-summary.json +++ b/docs/llms/docs-benchmark-summary.json @@ -1,38 +1,38 @@ { "version": 1, - "generated_at": "2026-01-23T20:54:53.246Z", + "generated_at": "2026-01-24T00:05:38.779Z", "by_language": { "csharp": { "modes": { "docs": { - "hash": "899b719429476f143199a527ed8f2581ee4b9a9915442fc98f13315db1fc2b27", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "models": { "GPT-5": { "categories": { "basics": { "tasks": 12, "total_tests": 27, - "passed_tests": 24, - "pass_pct": 88.888885, + "passed_tests": 26, + "pass_pct": 96.296295, "task_pass_equiv": 11.0, "task_pass_pct": 91.66667 }, "schema": { "tasks": 10, - "total_tests": 34, - "passed_tests": 22, - "pass_pct": 64.70588, - "task_pass_equiv": 6.366667, - "task_pass_pct": 63.666664 + "total_tests": 30, + "passed_tests": 24, + "pass_pct": 80.0, + "task_pass_equiv": 7.1666665, + "task_pass_pct": 71.666664 } }, "totals": { "tasks": 22, - "total_tests": 61, - "passed_tests": 46, - "pass_pct": 75.409836, - "task_pass_equiv": 17.366667, - "task_pass_pct": 78.9394 + "total_tests": 57, + "passed_tests": 50, + "pass_pct": 87.7193, + "task_pass_equiv": 18.166666, + "task_pass_pct": 82.57576 } } } @@ -42,40 +42,40 @@ "rust": { "modes": { "docs": { - "hash": "01db7170253b69e00ab59c502f6af64d11383ec89e6f39217732572bfc3cee9e", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "models": { "GPT-5": { "categories": { "basics": { "tasks": 12, - "total_tests": 27, - "passed_tests": 5, - "pass_pct": 18.518518, - "task_pass_equiv": 1.3333334, - "task_pass_pct": 11.111112 + "total_tests": 24, + "passed_tests": 4, + "pass_pct": 16.666666, + "task_pass_equiv": 1.0833334, + "task_pass_pct": 9.027779 }, "schema": { "tasks": 10, - "total_tests": 30, - "passed_tests": 4, - "pass_pct": 13.333333, - "task_pass_equiv": 1.25, - "task_pass_pct": 12.5 + "total_tests": 34, + "passed_tests": 8, + "pass_pct": 23.529411, + "task_pass_equiv": 2.05, + "task_pass_pct": 20.5 } }, "totals": { "tasks": 22, - "total_tests": 57, - "passed_tests": 9, - "pass_pct": 15.789474, - "task_pass_equiv": 2.5833335, - "task_pass_pct": 11.742425 + "total_tests": 58, + "passed_tests": 12, + "pass_pct": 20.689655, + "task_pass_equiv": 3.1333334, + "task_pass_pct": 14.242424 } } } }, "rustdoc_json": { - "hash": "e6a5a5374022e3250e0df2cb5e4f7e4777081f474a0ebc4e99302be42151ef59", + "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "models": { "GPT-5": { "categories": {