diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 349624688f2..f70b677ac0c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -608,6 +608,9 @@ jobs: exit 1 } + - name: Hydrate Unity SDK DLLs + run: cargo ci dlls + - name: Check Unity meta files uses: DeNA/unity-meta-check@v3 with: @@ -630,10 +633,7 @@ jobs: - name: Patch com.clockworklabs.spacetimedbsdk dependency in manifest.json working-directory: demo/Blackholio/client-unity/Packages run: | - # Replace the com.clockworklabs.spacetimedbsdk dependency with the current branch. - # Note: Pointing to a local directory does not work, because our earlier steps nuke our meta files, which then causes Unity to not properly respect the DLLs (e.g. - # codegen does not work properly). - yq e -i '.dependencies["com.clockworklabs.spacetimedbsdk"] = "https://github.com/clockworklabs/SpacetimeDB.git?path=sdks/csharp#${{ github.head_ref }}"' manifest.json + yq e -i '.dependencies["com.clockworklabs.spacetimedbsdk"] = "file:../../../../sdks/csharp"' manifest.json cat manifest.json - uses: actions/cache@v3 diff --git a/docs/docs/00100-intro/00100-getting-started/00400-key-architecture.md b/docs/docs/00100-intro/00100-getting-started/00400-key-architecture.md index ce1c8a7c322..63eff91f7b0 100644 --- a/docs/docs/00100-intro/00100-getting-started/00400-key-architecture.md +++ b/docs/docs/00100-intro/00100-getting-started/00400-key-architecture.md @@ -185,7 +185,7 @@ spacetimedb.reducer('hello', (ctx) => { } }); -spacetimedb.reducer('world', (ctx) => { +const world = spacetimedb.reducer('world', (ctx) => { clearAllTables(ctx); // ... }); diff --git a/docs/llms/docs-benchmark-analysis.md b/docs/llms/docs-benchmark-analysis.md index 1ea6e1ad6b1..f1ac99a24bc 100644 --- a/docs/llms/docs-benchmark-analysis.md +++ b/docs/llms/docs-benchmark-analysis.md @@ -4,394 +4,370 @@ Generated from: `/__w/SpacetimeDB/SpacetimeDB/tools/xtask-llm-benchmark/../../do ## Summary -- **Total failures analyzed**: 31 +- **Total failures analyzed**: 35 --- -# Analysis of SpacetimeDB Benchmark Test Failures +# SpacetimeDB Benchmark Failures Analysis ## Rust / rustdoc_json Failures -### Compile/Publish Errors (3 Failures) - -#### 1. **t_002_scheduled_table** -- **Generated Code**: - ```rust - #[table(name = tick_timer, schedule(column = scheduled_at, reducer = tick))] - pub struct TickTimer { - #[primary_key] - #[auto_inc] - scheduled_id: u64, - scheduled_at: ScheduleAt, - } - ``` - -- **Golden Example**: - ```rust - #[table(name = tick_timer, scheduled(tick))] - pub struct TickTimer { - #[primary_key] - #[auto_inc] - pub scheduled_id: u64, - pub scheduled_at: ScheduleAt, - } - ``` - -- **Error**: `publish_error: spacetime publish failed (exit=1)` - -- **Explanation**: - The LLM used incorrect syntax for the `scheduled` attribute. It should be `scheduled(tick)` instead of `schedule(column = scheduled_at, reducer = tick)`. - -- **Root Cause**: The documentation may not clearly explain the syntax for the `scheduled` attribute. - -- **Recommendation**: Update documentation to emphasize that the `scheduled` attribute must be structured as `scheduled(reducer_name)`. - ---- - -#### 2. **t_003_struct_in_table** -- **Generated Code**: - ```rust - #[spacetimedb::table(name = entity)] - pub struct Entity { - #[primary_key] - id: i32, - pos: Position, - } - ``` - -- **Golden Example**: - ```rust - #[table(name = entity)] - pub struct Entity { - #[primary_key] - pub id: i32, - pub pos: Position, - } - ``` - -- **Error**: `publish_error: spacetime publish failed (exit=1)` +### Compile/Publish Errors -- **Explanation**: - The LLM did not use `pub` for struct fields which is required for visibility in SpacetimeDB. +#### t_002_scheduled_table +**The generated code:** +```rust +use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt}; -- **Root Cause**: The visibility rules for struct fields in Rust may need clearer explanation in the documentation. +#[table(name = tick_timer, scheduled(reducer = tick, column = scheduled_at))] +pub struct TickTimer { + #[primary_key] + #[auto_inc] + scheduled_id: u64, + scheduled_at: ScheduleAt, +} -- **Recommendation**: Include specific examples indicating that all fields in SpacetimeDB tables should be public. - ---- - -#### 3. **t_017_scheduled_columns** -- **Generated Code**: - ```rust - #[reducer(init)] - pub fn init(ctx: &ReducerContext) { - if ctx.db.tick_timer().count() == 0 { - ctx.db.tick_timer().insert(TickTimer { - scheduled_id: 0, - scheduled_at: ScheduleAt::repeat_micros(50_000), - }); - } - } - ``` - -- **Golden Example**: - ```rust - #[reducer(init)] - pub fn init(ctx: &ReducerContext) { - let every_50ms: ScheduleAt = Duration::from_millis(50).into(); +#[reducer(init)] +pub fn init(ctx: &ReducerContext) { + if ctx.db.tick_timer().count() == 0 { ctx.db.tick_timer().insert(TickTimer { scheduled_id: 0, - scheduled_at: every_50ms, + scheduled_at: ScheduleAt::repeat(50_000u64), }); } - ``` - -- **Error**: `publish_error: spacetime publish failed (exit=1)` - -- **Explanation**: The method for initializing `scheduled_at` is incorrect. Instead of using `repeat_micros()`, the code should convert a `Duration` to `ScheduleAt`. - -- **Root Cause**: Misunderstanding of the proper way to initialize scheduled columns could be reflected in lacking documentation details. - -- **Recommendation**: Clarify the documentation regarding initializing `ScheduleAt`, emphasizing conversion from `Duration`. - ---- - -### Other Failures (2 Failures) - -#### 4. **t_016_sum_type_columns** -- **Generated Code**: - ```rust - #[spacetimedb::table(name = drawing)] - pub struct Drawing { - #[primary_key] - id: i32, - a: Shape, - b: Shape, - } - ``` - -- **Golden Example**: - ```rust - #[table(name = drawing)] - pub struct Drawing { - #[primary_key] - pub id: i32, - pub a: Shape, - pub b: Shape, +} + +#[reducer] +pub fn tick(_ctx: &ReducerContext, _scheduled_id: u64) { +} +``` + +**The golden example:** +```rust +use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; +use std::time::Duration; + +#[table(name = tick_timer, scheduled(tick))] +pub struct TickTimer { + #[primary_key] + #[auto_inc] + pub scheduled_id: u64, + pub scheduled_at: ScheduleAt, +} + +#[reducer] +pub fn tick(_ctx: &ReducerContext, _row: TickTimer) -> Result<(), String> { + Ok(()) +} + +#[reducer(init)] +pub fn init(ctx: &ReducerContext) -> Result<(), String> { + ctx.db.tick_timer().insert(TickTimer { + scheduled_id: 0, + scheduled_at: ScheduleAt::Interval(Duration::from_millis(50).into()), + }); + Ok(()) +} +``` + +**The error:** `publish_error: spacetime publish failed (exit=1)` + +**Explain the difference:** +1. The LLM generated code uses incorrect API methods and structures. For instance, it uses `ScheduleAt::repeat(50_000u64)` instead of the correct `ScheduleAt::Interval(Duration::from_millis(50).into())`. +2. The reducer function `tick` should also return a `Result<(), String>` type as in the golden example. + +**Root cause:** The documentation may lack clarity on the specific types and return values required, especially around scheduling and reducers. + +**Recommendation:** +Update documentation to clarify the expected types and return values. Use the following: +- Make sure to specify correct API functions and structures in examples. + +#### t_017_scheduled_columns +**The generated code:** +```rust +use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt}; + +#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))] +pub struct TickTimer { + #[primary_key] + #[auto_inc] + scheduled_id: u64, + scheduled_at: ScheduleAt, +} + +#[reducer(init)] +pub fn init(ctx: &ReducerContext) { + let tbl = ctx.db.tick_timer(); + if tbl.count() == 0 { + tbl.insert(TickTimer { + scheduled_id: 0, + scheduled_at: ScheduleAt::RepeatMicros(50_000), + }); } - ``` - -- **Error**: Errors regarding tables not found. - -- **Explanation**: Missing the `pub` attribute on struct fields results in failure to compile. - -- **Root Cause**: Lack of clarity on the use of visibility attributes (`pub`) in struct definitions. - -- **Recommendation**: Revise documentation to instruct that fields must be public to work within SpacetimeDB. +} + +#[reducer] +pub fn tick(_ctx: &ReducerContext, _row: TickTimer) { +} +``` + +**The golden example:** +```rust +use spacetimedb::{reducer, table, ReducerContext, ScheduleAt, Table}; +use std::time::Duration; + +#[table(name = tick_timer, scheduled(tick))] +pub struct TickTimer { + #[primary_key] + #[auto_inc] + pub scheduled_id: u64, + pub scheduled_at: ScheduleAt, +} + +#[reducer] +pub fn tick(_ctx: &ReducerContext, _schedule: TickTimer) { +} + +#[reducer(init)] +pub fn init(ctx: &ReducerContext) { + let every_50ms: ScheduleAt = Duration::from_millis(50).into(); + ctx.db.tick_timer().insert(TickTimer { + scheduled_id: 0, + scheduled_at: every_50ms, + }); +} +``` + +**The error:** `publish_error: spacetime publish failed (exit=1)` + +**Explain the difference:** +1. The LLM incorrectly uses an outdated or wrong structure for scheduling with `ScheduleAt::RepeatMicros(50_000)`. + +**Root cause:** There is likely a mismatch in expected vs. generated syntax, particularly around the scheduling configuration settings. + +**Recommendation:** +Reiterate the correct usage of scheduling in the updated documentation with ample examples. --- -#### 5. **t_020_ecs** -- **Generated Code**: - ```rust - #[spacetimedb::table(name = entity)] - pub struct Entity { - #[primary_key] - id: i32, - } - - #[spacetimedb::table(name = position)] - pub struct Position { - #[primary_key] - entity_id: i32, - x: i32, - y: i32, - } - ``` - -- **Golden Example**: - ```rust - #[table(name = entity)] - pub struct Entity { - #[primary_key] - pub id: i32, - } - - #[table(name = position)] - pub struct Position { - #[primary_key] - pub entity_id: i32, - pub x: i32, - pub y: i32, - } - ``` - -- **Error**: Errors regarding tables not found. - -- **Explanation**: Missing the `pub` attribute leads to the struct not being properly registered with SpacetimeDB. - -- **Root Cause**: Similar to previous errors, the need for public access to struct fields is unclear. - -- **Recommendation**: Ensure documentation explicitly states that public access is necessary for all fields in SpacetimeDB structs. +### Other Failures + +#### t_001_basic_tables +**The generated code:** +```rust +use spacetimedb::ReducerContext; + +#[spacetimedb::table(name = user)] +pub struct User { + #[primary_key] + id: i32, + name: String, + age: i32, + active: bool, +} + +#[spacetimedb::table(name = product)] +pub struct Product { + #[primary_key] + id: i32, + title: String, + price: f32, + in_stock: bool, +} + +#[spacetimedb::table(name = note)] +pub struct Note { + #[primary_key] + id: i32, + body: String, + rating: i64, + pinned: bool, +} + +#[spacetimedb::reducer(init)] +pub fn init(_ctx: &ReducerContext) {} +``` + +**The golden example:** +```rust +use spacetimedb::table; + +#[table(name = user)] +pub struct User { + #[primary_key] + pub id: i32, + pub name: String, + pub age: i32, + pub active: bool, +} + +#[table(name = product)] +pub struct Product { + #[primary_key] + pub id: i32, + pub title: String, + pub price: f32, + pub in_stock: bool, +} + +#[table(name = note)] +pub struct Note { + #[primary_key] + pub id: i32, + pub body: String, + pub rating: i64, + pub pinned: bool, +} +``` + +**The error:** `schema_parity: reducers differ - expected [], got ["init()"]` + +**Explain the difference:** The generated code does not include any public fields in the struct definitions, which is a requirement for proper table mapping. + +**Root cause:** Public visibility for struct fields may not have been emphasized in the existing documentation. + +**Recommendation:** +Explain the importance of making struct fields public in documentation examples to prevent mismatches during schema checks. --- -## Rust / docs Failures (22 total) - -### Timeout Issues (8 Failures) - -- **Failures**: Various tasks timed out, indicating potential performance or configuration issues. - -- **Root Cause**: Specifics of timeout settings and performance optimization strategies should be more explicit in the documentation. - -- **Recommendation**: Include guidelines on optimizing performance for long-running tasks or emphasize best practices for structuring queries and data handling. +### Insert and CRUD Related Tests + +#### t_004_insert +**The generated code:** +```rust +use spacetimedb::{reducer, table, ReducerContext, Table}; + +#[table(name = user)] +pub struct User { + #[primary_key] + id: i32, + name: String, + age: i32, + active: bool, +} + +#[reducer] +pub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) { + ctx.db.user().insert(User { id, name, age, active }); +} +``` + +**The golden example:** +```rust +use spacetimedb::{reducer, table, ReducerContext, Table}; + +#[table(name = user)] +pub struct User { + #[primary_key] + pub id: i32, + pub name: String, + pub age: i32, + pub active: bool, +} + +#[reducer] +pub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) -> Result<(), String> { + ctx.db.user().insert(User { id, name, age, active }); + Ok(()) +} +``` + +**The error:** `data_parity_insert_user: spacetime sql failed: Error: no such table: user.` + +**Explain the difference:** Missing the `pub` visibility specifier for struct fields and failing to return a result type in reducer functions. + +**Root cause:** Lack of emphasis on the need for public struct fields in the documentation and clear result type specifications. + +**Recommendation:** +Strengthen language around public field requirements and return types in reducer functions in the documentation. --- -### Other Failures (14 Failures) +## C# / docs Failures -#### 6. **t_000_empty_reducers** -- **Generated Code**: - ```rust - #[spacetimedb::reducer] - pub fn empty_reducer_no_args(_ctx: &spacetimedb::ReducerContext) { - } - ``` - -- **Golden Example**: - ```rust - #[reducer] - pub fn empty_reducer_no_args(ctx: &ReducerContext) -> Result<(), String> { - Ok(()) - } - ``` +### Timeout Issues -- **Error**: Schema-related errors due to missing return type and proper handling. - -- **Explanation**: Missing return type (`Result<(), String>`) was not implemented. - -- **Root Cause**: The documentation may not explicitly mention that reducers should return results. - -- **Recommendation**: Adjust the documentation to specify that reducer functions must include appropriate return types. - ---- - -#### 7. **t_001_basic_tables** -- **Generated Code**: - ```rust - #[spacetimedb::table(name = user)] - pub struct User { - #[primary_key] - id: i32, - name: String, - age: i32, - active: bool, - } - ``` - -- **Golden Example**: - ```rust - #[table(name = user)] - pub struct User { - #[primary_key] - pub id: i32, - pub name: String, - pub age: i32, - pub active: bool, - } - ``` - -- **Error**: Schema-related errors due to missing `pub` modifiers. - -- **Explanation**: Missing public access modifiers on struct fields prevented expected behavior. - -- **Root Cause**: Visibility rules may not have been adequately covered in the documentation. - -- **Recommendation**: Ensure the documentation includes examples with visibility modifiers. +#### t_020_ecs +*This failure is expected in both Rust and C#, highlighting a systemic timeout issue.* --- -### C# / docs Failures (4 total) +### Other Failures -#### Other Failures (4 Failures) +#### t_002_scheduled_table +**The generated code:** +```csharp +using SpacetimeDB; -#### 8. **t_014_elementary_columns** -- **Generated Code**: - ```csharp - [SpacetimeDB.Table(Name = "Primitive", Public = true)] - public partial struct Primitive - { - [SpacetimeDB.PrimaryKey] - public int Id; - public int Count; - ... - } - ``` - -- **Golden Example**: - ```csharp - [Table(Name = "Primitive")] - public partial struct Primitive +public static partial class Module +{ + [Table(Name = "TickTimer", Scheduled = nameof(Tick), ScheduledAt = nameof(TickTimer.ScheduledAt))] + public partial struct TickTimer { - [PrimaryKey] public int Id; - public int Count; - ... + [PrimaryKey, AutoInc] public ulong ScheduledId; + public ScheduleAt ScheduledAt; } - ``` -- **Error**: Table not found during sql operations. - -- **Explanation**: The `Public` attribute's use was incorrect; it's not necessary in the struct definition. - -- **Root Cause**: Confusion over the purpose and necessity of attributes. - -- **Recommendation**: Update documentation to clarify attributes' roles in table definitions, removing unnecessary ones for struct exposure. - ---- + [Reducer] + public static void Tick(ReducerContext ctx, TickTimer timer) { } -#### 9. **t_016_sum_type_columns** -- **Generated Code**: - ```csharp - [SpacetimeDB.Table(Name = "Drawing", Public = true)] - public partial struct Drawing + [Reducer(ReducerKind.Init)] + public static void Init(ReducerContext ctx) { - [SpacetimeDB.PrimaryKey] - public int Id; - } - ``` - -- **Golden Example**: - ```csharp - [Table(Name = "Drawing")] - public partial struct Drawing - { - [PrimaryKey] public int Id; + var interval = new TimeDuration { Microseconds = 50_000 }; + ctx.Db.TickTimer.Insert(new TickTimer + { + ScheduledAt = new ScheduleAt.Interval(interval) + }); } - ``` - -- **Error**: Table not found during sql operations. - -- **Explanation**: Similar to the previous failure, the `Public` attribute was misapplied. +} +``` -- **Root Cause**: Misalignment between understood attribute requirements and actual usage. +**The expected example:** +```csharp +using SpacetimeDB; -- **Recommendation**: Further clarification of when and where to apply attributes in C# constructs related to SpacetimeDB. - ---- - -#### 10. **t_017_scheduled_columns** -- **Generated Code**: - ```csharp - [Table(Name = "TickTimer", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))] +public static partial class Module +{ + [Table(Name = "TickTimer", Scheduled = nameof(Tick), ScheduledAt = nameof(TickTimer.ScheduledAt))] public partial struct TickTimer { - [PrimaryKey, AutoInc] - public ulong ScheduledId; + [PrimaryKey, AutoInc] public ulong ScheduledId; public ScheduleAt ScheduledAt; } - ``` -- **Golden Example**: - ```csharp - [Table(Name = "TickTimer", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))] - public partial struct TickTimer + [Reducer] + public static void Tick(ReducerContext ctx, TickTimer _row) { } + + [Reducer(ReducerKind.Init)] + public static void Init(ReducerContext ctx) { - [PrimaryKey, AutoInc] public ulong ScheduledId; - public ScheduleAt ScheduledAt; + var interval = new TimeDuration { Microseconds = 50_000 }; + ctx.Db.TickTimer.Insert(new TickTimer + { + ScheduledId = 0, + ScheduledAt = new ScheduleAt.Interval(interval) + }); } - ``` +} +``` -- **Error**: Table not found during sql operations. +**The error:** `publish_error: 500 Internal Server Error` -- **Explanation**: The field definitions and relationships were incorrectly configured. +**Explain the difference:** In the generated code, the scheduled fields are improperly initialized, and the reducer method signatures do not entirely match expectations regarding types and method names. -- **Root Cause**: Possible gaps in documentation regarding definitions of scheduled columns and expected real structures. +**Root cause:** Possibly unclear examples in the documentation regarding the scheduled fields and method signature conventions. -- **Recommendation**: Revise documentation to ensure clear expectations about table configuration and proper struct setup. +**Recommendation:** +Revise documentation examples to ensure all aspects of the scheduled field implementations and reducer methods are included, particularly concerning required fields. --- -#### 11. **t_020_ecs** -- **Generated Code**: - ```csharp - [SpacetimeDB.Table(Name = "Entity", Public = true)] - public partial struct Entity { [SpacetimeDB.PrimaryKey] public int Id; } - ``` - -- **Golden Example**: - ```csharp - [Table(Name = "Entity")] - public partial struct Entity { [PrimaryKey] public int Id; } - ``` - -- **Error**: Errors related to missing tables. - -- **Explanation**: Public attributes were misused in creating struct definitions for the tables. - -- **Root Cause**: Attribute usage may be causing confusion in use cases. - -- **Recommendation**: Ensure documentation includes proper usage guidelines for attributes in defining entities. - ---- +### Conclusion +The main failures across languages predominantly stem from three issues: +1. Lack of clarity on public struct field visibility. +2. Missing result type requirements in reducer methods. +3. Incorrect method signatures and scheduling configurations. -By addressing the aforementioned discrepancies and gaps in documentation, developers can improve their implementation of SpacetimeDB, leading to smoother integrations and reduced error rates during execution. +Specific recommendations center around **enhancing documentation with clearer examples**, **emphasizing visibility requirements**, and detailing the **return types for functions**. This will help streamline the development process and mitigate these common errors in future benchmarks. diff --git a/docs/llms/docs-benchmark-comment.md b/docs/llms/docs-benchmark-comment.md index 37272389e0d..d42a62c2d60 100644 --- a/docs/llms/docs-benchmark-comment.md +++ b/docs/llms/docs-benchmark-comment.md @@ -2,16 +2,16 @@ | Language | Mode | Category | Tests Passed | Task Pass % | |----------|------|----------|--------------|-------------| -| Rust | rustdoc_json | basics | 25/27 | 83.3% ⬆️ +9.0% | -| Rust | rustdoc_json | schema | 26/34 | 75.3% ⬆️ +10.0% | -| Rust | rustdoc_json | **total** | 51/61 | **79.7%** ⬆️ +9.5% | -| Rust | docs | basics | 5/27 | 11.1% | -| Rust | docs | schema | 0/14 | 0.0% ⬇️ -20.5% | -| Rust | docs | **total** | 5/41 | **6.1%** ⬇️ -9.3% | -| C# | docs | basics | 27/27 | 100.0% | -| C# | docs | schema | 25/34 | 73.7% | -| C# | docs | **total** | 52/61 | **88.0%** | +| Rust | rustdoc_json | basics | 22/27 | 74.3% | +| Rust | rustdoc_json | schema | 26/34 | 75.3% | +| Rust | rustdoc_json | **total** | 48/61 | **74.8%** | +| Rust | docs | basics | 4/24 | 9.0% ⬇️ -2.1% | +| Rust | docs | schema | 8/34 | 20.5% ⬆️ +8.0% | +| Rust | docs | **total** | 12/58 | **14.2%** ⬆️ +2.5% | +| C# | docs | basics | 26/27 | 91.7% | +| C# | docs | schema | 24/30 | 71.7% ⬆️ +8.0% | +| C# | docs | **total** | 50/57 | **82.6%** ⬆️ +3.6% | _Compared against master branch baseline_ -Generated at: 2026-01-23T20:21:59.670Z +Generated at: 2026-01-24T00:05:38.779Z diff --git a/docs/llms/docs-benchmark-details.json b/docs/llms/docs-benchmark-details.json index 2c8b8a16a11..4982ed6acb5 100644 --- a/docs/llms/docs-benchmark-details.json +++ b/docs/llms/docs-benchmark-details.json @@ -19,7 +19,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\nfn empty_reducer_no_args(_ctx: &ReducerContext) {\n}\n\n#[spacetimedb::reducer]\nfn empty_reducer_with_int(_ctx: &ReducerContext, _count: i32) {\n}\n\n#[spacetimedb::reducer]\nfn empty_reducer_with_string(_ctx: &ReducerContext, _name: String) {\n}\n\n#[spacetimedb::reducer]\nfn empty_reducer_with_two_args(_ctx: &ReducerContext, _count: i32, _name: String) {\n}\n\n#[spacetimedb::reducer]\nfn empty_reducer_with_three_args(_ctx: &ReducerContext, _active: bool, _ratio: f32, _label: String) {\n}", + "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, _count: i32) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, _name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, _count: i32, _name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, _active: bool, _ratio: f32, _label: String) {\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-000-empty-reducers-golden", @@ -35,15 +35,15 @@ "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41115", + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:09:12.144740177Z", - "finished_at": "2026-01-23T20:09:46.065291337Z" + "started_at": "2026-01-23T23:54:12.885499372Z", + "finished_at": "2026-01-23T23:54:29.927523908Z" }, "t_001_basic_tables": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -52,8 +52,8 @@ "golden_published": true, "model_name": "GPT-5", "total_tests": 1, - "passed_tests": 1, - "llm_output": "use spacetimedb::table;\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", + "passed_tests": 0, + "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(_ctx: &ReducerContext) {}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-001-basic-tables-golden", @@ -62,22 +62,27 @@ "work_dir_llm": "target/llm-runs/basics/t_001_basic_tables/rust/server/gpt-5/llm", "scorer_details": { "schema_parity": { - "pass": true, - "partial": 1.0, + "pass": false, + "partial": 0.0, "notes": { "golden_db": "basics-t-001-basic-tables-golden", "llm_db": "basics-t-001-basic-tables-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41115", + "reducers_diff": { + "only_golden": [], + "only_llm": [ + "init()" + ] + }, + "reducers_equal": false, + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:09:14.464369753Z", - "finished_at": "2026-01-23T20:09:44.014749235Z" + "started_at": "2026-01-23T23:54:12.884839011Z", + "finished_at": "2026-01-23T23:54:44.364239410Z" }, "t_002_scheduled_table": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -87,7 +92,7 @@ "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, ScheduleAt, Table};\n\n#[table(name = tick_timer, schedule(column = scheduled_at, reducer = tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext) {}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, scheduled(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat(50_000u64),\n });\n }\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext, _scheduled_id: u64) {\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-002-scheduled-table-golden", @@ -99,24 +104,24 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling serde v1.0.228\n Compiling zerocopy v0.8.33\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling find-msvc-tools v0.1.8\n Compiling bitflags v2.10.0\n Compiling nohash-hasher v0.2.0\n Compiling anyhow v1.0.100\n Compiling thiserror v1.0.69\n Compiling zmij v1.0.16\n Compiling keccak v0.1.5\n Compiling heck v0.4.1\n Compiling arrayvec v0.7.6\n Compiling bytes v1.11.0\n Compiling humantime v2.3.0\n Compiling convert_case v0.4.0\n Compiling itoa v1.0.17\n Compiling second-stack v0.3.5\n Compiling serde_json v1.0.149\n Compiling constant_time_eq v0.4.2\n Compiling getrandom v0.2.17\n Compiling smallvec v1.15.1\n Compiling hex v0.4.3\n Compiling cc v1.2.54\n Compiling bytemuck v1.24.0\n Compiling arrayref v0.3.9\n Compiling spacetimedb-lib v1.11.1\n Compiling itertools v0.12.1\n Compiling memchr v2.7.6\n Compiling log v0.4.29\n Compiling rand_core v0.6.4\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling blake3 v1.8.3\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(column = scheduled_at, reducer = tick))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:18:36\n |\n18 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:17:15\n |\n17 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:18:16\n |\n18 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:20:39\n |\n20 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0422, E0599.\nFor more information about an error, try `rustc --explain E0422`.\nerror: could not compile `spacetime-module` (lib) due to 5 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling either v1.15.0\n Compiling find-msvc-tools v0.1.8\n Compiling shlex v1.3.0\n Compiling serde v1.0.228\n Compiling zerocopy v0.8.33\n Compiling bitflags v2.10.0\n Compiling anyhow v1.0.100\n Compiling nohash-hasher v0.2.0\n Compiling thiserror v1.0.69\n Compiling zmij v1.0.16\n Compiling heck v0.4.1\n Compiling arrayvec v0.7.6\n Compiling convert_case v0.4.0\n Compiling humantime v2.3.0\n Compiling keccak v0.1.5\n Compiling bytes v1.11.0\n Compiling arrayref v0.3.9\n Compiling itoa v1.0.17\n Compiling hex v0.4.3\n Compiling serde_json v1.0.149\n Compiling getrandom v0.2.17\n Compiling smallvec v1.15.1\n Compiling itertools v0.12.1\n Compiling cc v1.2.54\n Compiling second-stack v0.3.5\n Compiling constant_time_eq v0.4.2\n Compiling spacetimedb-lib v1.11.1\n Compiling rand_core v0.6.4\n Compiling bytemuck v1.24.0\n Compiling generic-array v0.14.7\n Compiling memchr v2.7.6\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling blake3 v1.8.3\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling sha3 v0.10.8\n Compiling decorum v0.3.1\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm)\nerror: expected `at`\n --> src/lib.rs:4:38\n |\n4 | #[table(name = tick_timer, scheduled(reducer = tick, column = scheduled_at))]\n | ^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:15:36\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:15\n |\n14 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:15:16\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:17:39\n |\n17 | scheduled_at: ScheduleAt::repeat(50_000u64),\n | ^^^^^^ variant or associated item not found in `ScheduleAt`\n\nSome errors have detailed explanations: E0422, E0599.\nFor more information about an error, try `rustc --explain E0422`.\nerror: could not compile `spacetime-module` (lib) due to 5 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:09:35.992436277Z", - "finished_at": "2026-01-23T20:10:39.770857477Z" + "started_at": "2026-01-23T23:54:12.885646580Z", + "finished_at": "2026-01-23T23:55:20.697821642Z" }, "t_003_struct_in_table": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", "task": "t_003_struct_in_table", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", "total_tests": 1, - "passed_tests": 0, - "llm_output": "use spacetimedb::ReducerContext;\n\n#[derive(spacetimedb::SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn add_entity(ctx: &ReducerContext, id: i32, x: i32, y: i32) {\n let pos = Position { x, y };\n ctx.db.entity().insert(Entity { id, pos });\n}", + "passed_tests": 1, + "llm_output": "use spacetimedb::{table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Position {\n pub x: i32,\n pub y: i32,\n}\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n pub id: i32,\n pub pos: Position,\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-003-struct-in-table-golden", @@ -124,18 +129,23 @@ "work_dir_golden": "target/llm-runs/basics/t_003_struct_in_table/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_003_struct_in_table/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { - "pass": false, - "partial": 0.0, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling unicode-ident v1.0.22\n Compiling quote v1.0.44\n Compiling version_check v0.9.5\n Compiling typenum v1.19.0\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling zerocopy v0.8.33\n Compiling serde v1.0.228\n Compiling shlex v1.3.0\n Compiling either v1.15.0\n Compiling find-msvc-tools v0.1.8\n Compiling anyhow v1.0.100\n Compiling nohash-hasher v0.2.0\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling bytes v1.11.0\n Compiling heck v0.4.1\n Compiling humantime v2.3.0\n Compiling zmij v1.0.16\n Compiling keccak v0.1.5\n Compiling arrayvec v0.7.6\n Compiling convert_case v0.4.0\n Compiling hex v0.4.3\n Compiling bytemuck v1.24.0\n Compiling second-stack v0.3.5\n Compiling arrayref v0.3.9\n Compiling getrandom v0.2.17\n Compiling spacetimedb-lib v1.11.1\n Compiling itoa v1.0.17\n Compiling itertools v0.12.1\n Compiling constant_time_eq v0.4.2\n Compiling smallvec v1.15.1\n Compiling serde_json v1.0.149\n Compiling log v0.4.29\n Compiling memchr v2.7.6\n Compiling rand_core v0.6.4\n Compiling scoped-tls v1.0.1\n Compiling cc v1.2.54\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling http v1.4.0\n Compiling syn v2.0.114\n Compiling blake3 v1.8.3\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/basics/t_003_struct_in_table/rust/server/gpt-5/llm)\nerror[E0599]: no method named `insert` found for reference `&entity__TableHandle` in the current scope\n --> src/lib.rs:20:21\n |\n20 | ctx.db.entity().insert(Entity { id, pos });\n | ^^^^^^\n |\n = help: items from traits can only be used if the trait is in scope\nhelp: trait `Table` which provides `insert` is implemented but not in scope; perhaps you want to import it\n |\n 2 + use spacetimedb::Table;\n |\nhelp: there is a method `try_insert` with a similar name\n |\n20 | ctx.db.entity().try_insert(Entity { id, pos });\n | ++++\n\nFor more information about this error, try `rustc --explain E0599`.\nerror: could not compile `spacetime-module` (lib) due to 1 previous error\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", - "phase": "build_or_publish" + "golden_db": "basics-t-003-struct-in-table-golden", + "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:37093", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:09:14.464130029Z", - "finished_at": "2026-01-23T20:09:54.314351498Z" + "started_at": "2026-01-23T23:54:12.885049901Z", + "finished_at": "2026-01-23T23:54:44.022438380Z" }, "t_004_insert": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -144,8 +154,8 @@ "golden_published": true, "model_name": "GPT-5", "total_tests": 2, - "passed_tests": 2, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().insert(User { id, name, age, active });\n}", + "passed_tests": 1, + "llm_output": "use spacetimedb::{reducer, table, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn insert_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n ctx.db.user().insert(User { id, name, age, active });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-004-insert-golden", @@ -153,25 +163,6 @@ "work_dir_golden": "target/llm-runs/basics/t_004_insert/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_004_insert/rust/server/gpt-5/llm", "scorer_details": { - "data_parity_insert_user": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - "Alice", - 30, - true - ], - "golden_db": "basics-t-004-insert-golden", - "golden_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "llm_db": "basics-t-004-insert-gpt-5-llm", - "llm_out": "id | name | age | active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "insert_user", - "server": "http://127.0.0.1:41115" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -180,15 +171,23 @@ "llm_db": "basics-t-004-insert-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41115", + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } + }, + "data_parity_insert_user": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `user`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37093/v1/database/c2002cd40e701250226f73cd286f21d0fabecee66409ea7072d8547d122d732f/sql)\n", + "phase": "sql_golden" + } } }, "vendor": "openai", - "started_at": "2026-01-23T20:09:14.983974693Z", - "finished_at": "2026-01-23T20:09:35.992396086Z" + "started_at": "2026-01-23T23:54:12.885355239Z", + "finished_at": "2026-01-23T23:54:36.594727744Z" }, "t_005_update": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -198,7 +197,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n let row = User { id, name, age, active };\n ctx.db.user().id().update(row);\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n let user_tbl = ctx.db.user();\n if user_tbl.id().find(id).is_some() {\n let row = User { id, name, age, active };\n let _ = user_tbl.id().update(row);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -206,13 +205,6 @@ "work_dir_golden": "target/llm-runs/basics/t_005_update/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_005_update/rust/server/gpt-5/llm", "scorer_details": { - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, "data_parity_update_user": { "pass": true, "partial": 1.0, @@ -229,7 +221,7 @@ "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "query": "SELECT id, name, age, active FROM user WHERE id=1", "reducer": "update_user", - "server": "http://127.0.0.1:41115" + "server": "http://127.0.0.1:37093" } }, "schema_parity": { @@ -240,15 +232,22 @@ "llm_db": "basics-t-005-update-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41115", + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } + }, + "seed_users_row": { + "pass": true, + "partial": 1.0, + "notes": { + "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" + } } }, "vendor": "openai", - "started_at": "2026-01-23T20:08:57.792213429Z", - "finished_at": "2026-01-23T20:09:53.091667873Z" + "started_at": "2026-01-23T23:54:29.927557906Z", + "finished_at": "2026-01-23T23:55:05.452075958Z" }, "t_006_delete": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -275,6 +274,13 @@ "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" } }, + "seed_users_row": { + "pass": true, + "partial": 1.0, + "notes": { + "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -283,22 +289,15 @@ "llm_db": "basics-t-006-delete-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41115", + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } - }, - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } } }, "vendor": "openai", - "started_at": "2026-01-23T20:08:35.084889228Z", - "finished_at": "2026-01-23T20:09:14.643664761Z" + "started_at": "2026-01-23T23:54:43.492030285Z", + "finished_at": "2026-01-23T23:55:25.742698140Z" }, "t_007_crud": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -307,8 +306,8 @@ "golden_published": true, "model_name": "GPT-5", "total_tests": 4, - "passed_tests": 4, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, UniqueColumn};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n let users = ctx.db.user();\n\n let mut u1 = users.insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n\n let _u2 = users.insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n\n u1.name = \"Alice2\".to_string();\n u1.age = 31;\n u1.active = false;\n let _u1_updated = users.id().update(u1);\n\n users.id().delete(&2);\n}", + "passed_tests": 3, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer]\npub fn crud(ctx: &ReducerContext) {\n let mut u1 = ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n\n let _u2 = ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n\n u1.name = \"Alice2\".to_string();\n u1.age = 31;\n u1.active = false;\n let _u1_updated = ctx.db.user().id().update(u1);\n\n let id2 = 2i32;\n ctx.db.user().id().delete(&id2);\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -316,27 +315,12 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm", "scorer_details": { - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" - } - }, "crud_row_id1_parity": { - "pass": true, - "partial": 1.0, + "pass": false, + "partial": 0.0, "notes": { - "args": [], - "golden_db": "basics-t-007-crud-golden", - "golden_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "llm_db": "basics-t-007-crud-gpt-5-llm", - "llm_out": "id | name | age | active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT id, name, age, active FROM user WHERE id=1", - "reducer": "crud", - "server": "http://127.0.0.1:41115" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `user`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37093/v1/database/c200fd84162598d608587e2986747375700c8dd4786931b3439125fcb8d48289/sql)\n", + "phase": "sql_golden" } }, "crud_total_count_one": { @@ -356,15 +340,24 @@ "llm_db": "basics-t-007-crud-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41115", + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } + }, + "crud_row_id2_deleted": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" + } } }, "vendor": "openai", - "started_at": "2026-01-23T20:08:56.057273916Z", - "finished_at": "2026-01-23T20:09:48.569368820Z" + "started_at": "2026-01-23T23:54:12.885914392Z", + "finished_at": "2026-01-23T23:54:43.758962254Z" }, "t_008_index_lookup": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -374,7 +367,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(u) = ctx.db.user().id().find(id) {\n let User { id, name, .. } = u;\n ctx.db.result().id().delete(&id);\n ctx.db.result().insert(ResultRow { id, name });\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(user) = ctx.db.user().id().find(id) {\n ctx.db.result().id().delete(&id);\n ctx.db.result().insert(ResultRow { id: user.id, name: user.name });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -382,6 +375,19 @@ "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/rust/server/gpt-5/llm", "scorer_details": { + "schema_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "golden_db": "basics-t-008-index-lookup-golden", + "llm_db": "basics-t-008-index-lookup-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:37093", + "tables_diff": null, + "tables_equal": true + } + }, "seed_user_row": { "pass": true, "partial": 1.0, @@ -402,26 +408,13 @@ "llm_out": "id | name ----+--------- 1 | \"Alice\"", "query": "SELECT id, name FROM result WHERE id=1", "reducer": "lookup_user_name", - "server": "http://127.0.0.1:41115" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "basics-t-008-index-lookup-golden", - "llm_db": "basics-t-008-index-lookup-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41115", - "tables_diff": null, - "tables_equal": true + "server": "http://127.0.0.1:37093" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:09:14.643699808Z", - "finished_at": "2026-01-23T20:09:52.566672855Z" + "started_at": "2026-01-23T23:54:43.389799379Z", + "finished_at": "2026-01-23T23:55:22.299128028Z" }, "t_009_init": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -431,7 +424,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n let user = ctx.db.user();\n user.insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n user.insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) {\n let users = ctx.db.user();\n users.insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n users.insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -439,24 +432,6 @@ "work_dir_golden": "target/llm-runs/basics/t_009_init/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_009_init/rust/server/gpt-5/llm", "scorer_details": { - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" - } - }, - "init_seed_alice": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -465,7 +440,7 @@ "llm_db": "basics-t-009-init-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41115", + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } @@ -478,11 +453,29 @@ "expected": 2, "sql": "SELECT COUNT(*) AS n FROM user" } + }, + "init_seed_alice": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" + } + }, + "init_seed_bob": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" + } } }, "vendor": "openai", - "started_at": "2026-01-23T20:08:57.046090650Z", - "finished_at": "2026-01-23T20:09:23.724177316Z" + "started_at": "2026-01-23T23:54:36.594836950Z", + "finished_at": "2026-01-23T23:55:03.406964508Z" }, "t_010_connect": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -508,15 +501,15 @@ "llm_db": "basics-t-010-connect-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41115", + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:08:35.620982701Z", - "finished_at": "2026-01-23T20:09:12.144710803Z" + "started_at": "2026-01-23T23:54:12.885779579Z", + "finished_at": "2026-01-23T23:54:43.492006133Z" }, "t_011_helper_function": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -525,8 +518,8 @@ "golden_published": true, "model_name": "GPT-5", "total_tests": 3, - "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n ctx.db.result().insert(ResultRow { id, sum: add(a, b) });\n}", + "passed_tests": 2, + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[reducer]\nfn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n ctx.db.result().insert(ResultRow { id, sum: add(a, b) });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-011-helper-function-golden", @@ -534,50 +527,40 @@ "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/rust/server/gpt-5/llm", "scorer_details": { - "helper_func_sum_abs": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" + "golden_db": "basics-t-011-helper-function-golden", + "llm_db": "basics-t-011-helper-function-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:37093", + "tables_diff": null, + "tables_equal": true } }, "helper_func_sum_parity": { - "pass": true, - "partial": 1.0, + "pass": false, + "partial": 0.0, "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_db": "basics-t-011-helper-function-golden", - "golden_out": "id | sum ----+----- 1 | 5", - "llm_db": "basics-t-011-helper-function-gpt-5-llm", - "llm_out": "id | sum ----+----- 1 | 5", - "query": "SELECT id, sum FROM result WHERE id=1", - "reducer": "compute_sum", - "server": "http://127.0.0.1:41115" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `result`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37093/v1/database/c20077d8eb0a0abaac0c611ea4342c7aea3d0dd1ab6ac233f20ba9d0b1635306/sql)\n", + "phase": "sql_golden" } }, - "schema_parity": { + "helper_func_sum_abs": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "basics-t-011-helper-function-golden", - "llm_db": "basics-t-011-helper-function-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41115", - "tables_diff": null, - "tables_equal": true + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:09:23.724208808Z", - "finished_at": "2026-01-23T20:09:51.292524674Z" + "started_at": "2026-01-23T23:54:12.885202257Z", + "finished_at": "2026-01-23T23:54:43.389769168Z" }, "t_012_spacetime_product_type": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -587,7 +570,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow { id, value: Score { left, right } });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -595,6 +578,24 @@ "work_dir_golden": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_012_spacetime_product_type/rust/server/gpt-5/llm", "scorer_details": { + "product_type_row_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [ + 1, + 2, + 3 + ], + "golden_db": "schema-t-012-spacetime-product-type-golden", + "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", + "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", + "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", + "query": "SELECT id, value FROM result WHERE id=1", + "reducer": "set_score", + "server": "http://127.0.0.1:37093" + } + }, "product_type_row_count": { "pass": true, "partial": 1.0, @@ -612,33 +613,15 @@ "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41115", + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } - }, - "product_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_db": "schema-t-012-spacetime-product-type-golden", - "golden_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", - "llm_out": "id | value ----+----------------------- 1 | (left = 2, right = 3)", - "query": "SELECT id, value FROM result WHERE id=1", - "reducer": "set_score", - "server": "http://127.0.0.1:41115" - } } }, "vendor": "openai", - "started_at": "2026-01-23T20:08:32.450266496Z", - "finished_at": "2026-01-23T20:09:14.310539584Z" + "started_at": "2026-01-23T23:55:19.787335153Z", + "finished_at": "2026-01-23T23:55:43.342662434Z" }, "t_013_spacetime_sum_type": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -648,7 +631,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\nstruct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\nenum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = result)]\nstruct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[reducer]\nfn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow { id, value: Shape::Circle(radius) });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType)]\nstruct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\nenum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = result)]\nstruct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[spacetimedb::reducer]\nfn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().id().delete(&id);\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -673,7 +656,7 @@ "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41115", + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } @@ -692,13 +675,13 @@ "llm_out": "id | value ----+--------------- 1 | (Circle = 10)", "query": "SELECT id, value FROM result WHERE id=1", "reducer": "set_circle", - "server": "http://127.0.0.1:41115" + "server": "http://127.0.0.1:37093" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:07:57.743148689Z", - "finished_at": "2026-01-23T20:08:32.450224286Z" + "started_at": "2026-01-23T23:55:20.697841019Z", + "finished_at": "2026-01-23T23:55:58.028620416Z" }, "t_014_elementary_columns": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -708,7 +691,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000i64,\n price: 1.5f32,\n ratio: 2.25f64,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[reducer]\nfn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".to_string(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-014-elementary-columns-golden", @@ -716,19 +699,6 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-014-elementary-columns-golden", - "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41115", - "tables_diff": null, - "tables_equal": true - } - }, "elementary_columns_row_parity": { "pass": true, "partial": 1.0, @@ -740,22 +710,35 @@ "llm_out": "id | count | total | price | ratio | active | name ----+-------+------------+-------+-------+--------+--------- 1 | 2 | 3000000000 | 1.5 | 2.25 | true | \"Alice\"", "query": "SELECT id, count, total, price, ratio, active, name FROM primitive WHERE id=1", "reducer": "seed", - "server": "http://127.0.0.1:41115" + "server": "http://127.0.0.1:37093" } }, - "elementary_columns_row_count": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" - } + "golden_db": "schema-t-014-elementary-columns-golden", + "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:37093", + "tables_diff": null, + "tables_equal": true + } + }, + "elementary_columns_row_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" + } } }, "vendor": "openai", - "started_at": "2026-01-23T20:08:30.379293605Z", - "finished_at": "2026-01-23T20:09:14.464089675Z" + "started_at": "2026-01-23T23:55:20.558344928Z", + "finished_at": "2026-01-23T23:55:44.434182296Z" }, "t_015_product_type_columns": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -765,7 +748,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\nstruct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType)]\nstruct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = profile)]\nstruct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[reducer]\nfn seed(ctx: &ReducerContext) {\n let home = Address { street: \"1 Main\".to_string(), zip: 11111 };\n let work = Address { street: \"2 Broad\".to_string(), zip: 22222 };\n let pos = Position { x: 7, y: 9 };\n match ctx.db.profile().id().find(1) {\n Some(mut row) => {\n row.home = home;\n row.work = work;\n row.pos = pos;\n ctx.db.profile().id().update(row);\n }\n None => {\n ctx.db.profile().insert(Profile { id: 1, home, work, pos });\n }\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let profile = ctx.db.profile();\n\n for row in profile.iter() {\n profile.id().delete(&row.id);\n }\n\n profile.insert(Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".to_string(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".to_string(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-015-product-type-columns-golden", @@ -773,20 +756,6 @@ "work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/rust/server/gpt-5/llm", "scorer_details": { - "product_type_columns_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [], - "golden_db": "schema-t-015-product-type-columns-golden", - "golden_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", - "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", - "query": "SELECT id, home, work, pos FROM profile WHERE id=1", - "reducer": "seed", - "server": "http://127.0.0.1:41115" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -795,7 +764,7 @@ "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41115", + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } @@ -808,11 +777,25 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" } + }, + "product_type_columns_row_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [], + "golden_db": "schema-t-015-product-type-columns-golden", + "golden_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", + "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", + "llm_out": "id | home | work | pos ----+----------------------------------+-----------------------------------+---------------- 1 | (street = \"1 Main\", zip = 11111) | (street = \"2 Broad\", zip = 22222) | (x = 7, y = 9)", + "query": "SELECT id, home, work, pos FROM profile WHERE id=1", + "reducer": "seed", + "server": "http://127.0.0.1:37093" + } } }, "vendor": "openai", - "started_at": "2026-01-23T20:07:57.743649083Z", - "finished_at": "2026-01-23T20:08:57.046048716Z" + "started_at": "2026-01-23T23:55:22.299174613Z", + "finished_at": "2026-01-23T23:55:51.516097076Z" }, "t_016_sum_type_columns": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -822,7 +805,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[derive(spacetimedb::SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(spacetimedb::SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n if ctx.db.drawing().id().find(1).is_none() {\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, SpacetimeType};\n\n#[derive(SpacetimeType)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -830,11 +813,19 @@ "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/gpt-5/llm", "scorer_details": { + "sum_type_columns_row_count": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37093/v1/database/c200baf6e05e127bb6c08d0dadc067cc47eff464f996cc516d17c02565c76231/sql)\n", + "phase": "sql" + } + }, "sum_type_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41115/v1/database/c200be63a5c073cb57163fbe420fbf96d0ca1c24fb78e5d67bbed19841d5eeee/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37093/v1/database/c20000f2f719d2473685f272970a6ecfa83b00dc746a1a773f4f8901a0e09aec/sql)\n", "phase": "sql_golden" } }, @@ -846,23 +837,15 @@ "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41115", + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } - }, - "sum_type_columns_row_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41115/v1/database/c2001257ca4ff5c673696324b8a29b10931896a2877a4425230bec1856988aad/sql)\n", - "phase": "sql" - } } }, "vendor": "openai", - "started_at": "2026-01-23T20:07:57.742955241Z", - "finished_at": "2026-01-23T20:08:35.620943048Z" + "started_at": "2026-01-23T23:55:24.734590745Z", + "finished_at": "2026-01-23T23:55:57.654789420Z" }, "t_017_scheduled_columns": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -872,7 +855,7 @@ "model_name": "GPT-5", "total_tests": 2, "passed_tests": 0, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n if ctx.db.tick_timer().count() == 0 {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::repeat_micros(50_000),\n });\n }\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table, ScheduleAt};\n\n#[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[reducer(init)]\npub fn init(ctx: &ReducerContext) {\n let tbl = ctx.db.tick_timer();\n if tbl.count() == 0 {\n tbl.insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::RepeatMicros(50_000),\n });\n }\n}\n\n#[reducer]\npub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -884,14 +867,14 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling serde_core v1.0.228\n Compiling heck v0.5.0\n Compiling cfg-if v1.0.4\n Compiling serde v1.0.228\n Compiling either v1.15.0\n Compiling shlex v1.3.0\n Compiling zerocopy v0.8.33\n Compiling find-msvc-tools v0.1.8\n Compiling nohash-hasher v0.2.0\n Compiling thiserror v1.0.69\n Compiling bitflags v2.10.0\n Compiling anyhow v1.0.100\n Compiling arrayvec v0.7.6\n Compiling zmij v1.0.16\n Compiling heck v0.4.1\n Compiling convert_case v0.4.0\n Compiling humantime v2.3.0\n Compiling bytes v1.11.0\n Compiling keccak v0.1.5\n Compiling constant_time_eq v0.4.2\n Compiling bytemuck v1.24.0\n Compiling second-stack v0.3.5\n Compiling smallvec v1.15.1\n Compiling getrandom v0.2.17\n Compiling hex v0.4.3\n Compiling spacetimedb-lib v1.11.1\n Compiling itertools v0.12.1\n Compiling cc v1.2.54\n Compiling serde_json v1.0.149\n Compiling itoa v1.0.17\n Compiling arrayref v0.3.9\n Compiling rand_core v0.6.4\n Compiling memchr v2.7.6\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling generic-array v0.14.7\n Compiling num-traits v0.2.19\n Compiling syn v2.0.114\n Compiling http v1.4.0\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling blake3 v1.8.3\n Compiling crypto-common v0.1.7\n Compiling block-buffer v0.10.4\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling ethnum v1.5.2\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:15:36\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:23:42\n |\n23 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:15\n |\n14 | if ctx.db.tick_timer().count() == 0 {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:15:16\n |\n15 | ctx.db.tick_timer().insert(TickTimer {\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `repeat_micros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:17:39\n |\n17 | scheduled_at: ScheduleAt::repeat_micros(50_000),\n | ^^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:23:8\n |\n 22 | #[reducer]\n | ---------- required by a bound introduced by this call\n 23 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `register_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:746:81\n |\n746 | pub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl Reducer<'a, A>) {\n | ^^^^^^^^^^^^^^ required by this bound in `register_reducer`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:23:8\n |\n22 | #[reducer]\n | ---------- required by a bound introduced by this call\n23 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `invoke_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:45:19\n |\n44 | pub fn invoke_reducer<'a, A: Args<'a>>(\n | -------------- required by a bound in this function\n45 | reducer: impl Reducer<'a, A>,\n | ^^^^^^^^^^^^^^ required by this bound in `invoke_reducer`\n\nSome errors have detailed explanations: E0277, E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0277`.\nerror: could not compile `spacetime-module` (lib) due to 8 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", + "error": "spacetime publish failed (exit=1)\n--- stderr ---\n Updating crates.io index\n Locking 72 packages to latest compatible versions\n Adding generic-array v0.14.7 (available: v0.14.9)\n Adding spacetimedb v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-macro v1.11.1 (available: v1.11.3)\n Adding spacetimedb-bindings-sys v1.11.1 (available: v1.11.3)\n Adding spacetimedb-lib v1.11.1 (available: v1.11.3)\n Adding spacetimedb-primitives v1.11.1 (available: v1.11.3)\n Adding spacetimedb-sats v1.11.1 (available: v1.11.3)\n Compiling proc-macro2 v1.0.106\n Compiling quote v1.0.44\n Compiling unicode-ident v1.0.22\n Compiling typenum v1.19.0\n Compiling version_check v0.9.5\n Compiling autocfg v1.5.0\n Compiling heck v0.5.0\n Compiling serde_core v1.0.228\n Compiling cfg-if v1.0.4\n Compiling either v1.15.0\n Compiling zerocopy v0.8.33\n Compiling shlex v1.3.0\n Compiling serde v1.0.228\n Compiling find-msvc-tools v0.1.8\n Compiling nohash-hasher v0.2.0\n Compiling bitflags v2.10.0\n Compiling thiserror v1.0.69\n Compiling anyhow v1.0.100\n Compiling zmij v1.0.16\n Compiling heck v0.4.1\n Compiling convert_case v0.4.0\n Compiling humantime v2.3.0\n Compiling arrayvec v0.7.6\n Compiling keccak v0.1.5\n Compiling bytes v1.11.0\n Compiling constant_time_eq v0.4.2\n Compiling bytemuck v1.24.0\n Compiling second-stack v0.3.5\n Compiling itoa v1.0.17\n Compiling getrandom v0.2.17\n Compiling serde_json v1.0.149\n Compiling hex v0.4.3\n Compiling rand_core v0.6.4\n Compiling arrayref v0.3.9\n Compiling smallvec v1.15.1\n Compiling itertools v0.12.1\n Compiling spacetimedb-lib v1.11.1\n Compiling memchr v2.7.6\n Compiling generic-array v0.14.7\n Compiling log v0.4.29\n Compiling scoped-tls v1.0.1\n Compiling num-traits v0.2.19\n Compiling cc v1.2.54\n Compiling syn v2.0.114\n Compiling http v1.4.0\n Compiling approx v0.3.2\n Compiling chrono v0.4.43\n Compiling decorum v0.3.1\n Compiling block-buffer v0.10.4\n Compiling crypto-common v0.1.7\n Compiling blake3 v1.8.3\n Compiling digest v0.10.7\n Compiling sha3 v0.10.8\n Compiling ppv-lite86 v0.2.21\n Compiling rand_chacha v0.3.1\n Compiling rand v0.8.5\n Compiling ethnum v1.5.2\n Compiling enum-as-inner v0.6.1\n Compiling thiserror-impl v1.0.69\n Compiling derive_more v0.99.20\n Compiling spacetimedb-primitives v1.11.1\n Compiling spacetimedb-bindings-sys v1.11.1\n Compiling spacetimedb-bindings-macro v1.11.1\n Compiling spacetimedb-sats v1.11.1\n Compiling spacetimedb v1.11.1\n Compiling spacetime-module v0.1.0 (/__w/SpacetimeDB/SpacetimeDB/target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm)\nerror: expected one of: `public`, `private`, `name`, `index`, `scheduled`\n --> src/lib.rs:4:28\n |\n4 | #[table(name = tick_timer, schedule(reducer = tick, column = scheduled_at))]\n | ^^^^^^^^\n\nerror[E0422]: cannot find struct, variant or union type `TickTimer` in this scope\n --> src/lib.rs:16:20\n |\n16 | tbl.insert(TickTimer {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0412]: cannot find type `TickTimer` in this scope\n --> src/lib.rs:24:42\n |\n24 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^^^^^^ not found in this scope\n\nerror[E0599]: no method named `tick_timer` found for struct `Local` in the current scope\n --> src/lib.rs:14:22\n |\n14 | let tbl = ctx.db.tick_timer();\n | ^^^^^^^^^^ method not found in `Local`\n\nerror[E0599]: no variant or associated item named `RepeatMicros` found for enum `ScheduleAt` in the current scope\n --> src/lib.rs:18:39\n |\n18 | scheduled_at: ScheduleAt::RepeatMicros(50_000),\n | ^^^^^^^^^^^^ variant or associated item not found in `ScheduleAt`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:24:8\n |\n 23 | #[reducer]\n | ---------- required by a bound introduced by this call\n 24 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `register_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:746:81\n |\n746 | pub fn register_reducer<'a, A: Args<'a>, I: FnInfo>(_: impl Reducer<'a, A>) {\n | ^^^^^^^^^^^^^^ required by this bound in `register_reducer`\n\nerror[E0277]: invalid reducer signature\n --> src/lib.rs:24:8\n |\n23 | #[reducer]\n | ---------- required by a bound introduced by this call\n24 | pub fn tick(_ctx: &ReducerContext, _row: TickTimer) {\n | ^^^^ this reducer signature is not valid\n |\n = help: the trait `Reducer<'_, _>` is not implemented for fn item `for<'a> fn(&'a ReducerContext, {type error}) {tick}`\n = note: \n = note: reducer signatures must match the following pattern:\n = note: `Fn(&ReducerContext, [T1, ...]) [-> Result<(), impl Display>]`\n = note: where each `Ti` type implements `SpacetimeType`.\n = note: \nnote: required by a bound in `invoke_reducer`\n --> /root/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/spacetimedb-1.11.1/src/rt.rs:45:19\n |\n44 | pub fn invoke_reducer<'a, A: Args<'a>>(\n | -------------- required by a bound in this function\n45 | reducer: impl Reducer<'a, A>,\n | ^^^^^^^^^^^^^^ required by this bound in `invoke_reducer`\n\nSome errors have detailed explanations: E0277, E0412, E0422, E0599.\nFor more information about an error, try `rustc --explain E0277`.\nerror: could not compile `spacetime-module` (lib) due to 7 previous errors\nError: command [\"cargo\", \"build\", \"--config=net.git-fetch-with-cli=true\", \"--target=wasm32-unknown-unknown\", \"--release\", \"--message-format=json-render-diagnostics\"] exited with code 101\n\n--- stdout ---\n", "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:07:57.742610502Z", - "finished_at": "2026-01-23T20:08:56.057254318Z" + "started_at": "2026-01-23T23:55:05.452109524Z", + "finished_at": "2026-01-23T23:56:06.104136012Z" }, "t_018_constraints": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -901,7 +884,7 @@ "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db\n .account()\n .insert(Account {\n id: 1,\n email: \"a@example.com\".to_string(),\n name: \"Alice\".to_string(),\n });\n ctx.db\n .account()\n .insert(Account {\n id: 2,\n email: \"b@example.com\".to_string(),\n name: \"Bob\".to_string(),\n });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let ids: Vec = ctx.db.account().iter().map(|r| r.id).collect();\n for id in ids {\n ctx.db.account().id().delete(&id);\n }\n ctx.db\n .account()\n .insert(Account { id: 1, email: \"a@example.com\".to_string(), name: \"Alice\".to_string() });\n ctx.db\n .account()\n .insert(Account { id: 2, email: \"b@example.com\".to_string(), name: \"Bob\".to_string() });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-018-constraints-golden", @@ -918,37 +901,37 @@ "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" } }, - "constraints_row_parity_after_seed": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "args": [], "golden_db": "schema-t-018-constraints-golden", - "golden_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", "llm_db": "schema-t-018-constraints-gpt-5-llm", - "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT id, email, name FROM account WHERE id=1", - "reducer": "seed", - "server": "http://127.0.0.1:41115" + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:37093", + "tables_diff": null, + "tables_equal": true } }, - "schema_parity": { + "constraints_row_parity_after_seed": { "pass": true, "partial": 1.0, "notes": { + "args": [], "golden_db": "schema-t-018-constraints-golden", + "golden_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", "llm_db": "schema-t-018-constraints-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41115", - "tables_diff": null, - "tables_equal": true + "llm_out": "id | email | name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", + "query": "SELECT id, email, name FROM account WHERE id=1", + "reducer": "seed", + "server": "http://127.0.0.1:37093" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:07:57.742384971Z", - "finished_at": "2026-01-23T20:08:30.379256146Z" + "started_at": "2026-01-23T23:54:44.364263416Z", + "finished_at": "2026-01-23T23:55:20.558298479Z" }, "t_019_many_to_many": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -958,7 +941,7 @@ "model_name": "GPT-5", "total_tests": 5, "passed_tests": 5, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n // Clear existing rows to ensure exactly the requested rows exist after seeding\n let membership_ids: Vec = ctx.db.membership().iter().map(|m| m.id).collect();\n for id in membership_ids {\n ctx.db.membership().id().delete(&id);\n }\n\n let user_ids: Vec = ctx.db.user().iter().map(|u| u.user_id).collect();\n for id in user_ids {\n ctx.db.user().user_id().delete(&id);\n }\n\n let group_ids: Vec = ctx.db.group().iter().map(|g| g.group_id).collect();\n for id in group_ids {\n ctx.db.group().group_id().delete(&id);\n }\n\n // Insert users\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n // Insert groups\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n // Insert memberships\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n for m in ctx.db.membership().iter() {\n ctx.db.membership().id().delete(&m.id);\n }\n for u in ctx.db.user().iter() {\n ctx.db.user().user_id().delete(&u.user_id);\n }\n for g in ctx.db.group().iter() {\n ctx.db.group().group_id().delete(&g.group_id);\n }\n\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-019-many-to-many-golden", @@ -966,59 +949,59 @@ "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/rust/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "m2m_has_1_10": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "schema-t-019-many-to-many-golden", - "llm_db": "schema-t-019-many-to-many-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41115", - "tables_diff": null, - "tables_equal": true + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" } }, - "m2m_has_1_20": { + "memberships_three_rows": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM membership" } }, - "m2m_has_2_20": { + "m2m_has_1_20": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" } }, - "memberships_three_rows": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM membership" + "golden_db": "schema-t-019-many-to-many-golden", + "llm_db": "schema-t-019-many-to-many-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:37093", + "tables_diff": null, + "tables_equal": true } }, - "m2m_has_1_10": { + "m2m_has_2_20": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:07:57.743323315Z", - "finished_at": "2026-01-23T20:08:57.792175612Z" + "started_at": "2026-01-23T23:55:03.407007070Z", + "finished_at": "2026-01-23T23:55:43.880918665Z" }, "t_020_ecs": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -1028,7 +1011,7 @@ "model_name": "GPT-5", "total_tests": 5, "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // Clear and insert entities\n for &id in &[1, 2] {\n if ctx.db.entity().id().find(id).is_some() {\n ctx.db.entity().id().delete(&id);\n }\n ctx.db.entity().insert(Entity { id });\n }\n\n // Clear existing positions/velocities for these entities and insert new values\n // Entity 1: pos=(0,0), vel=(1,0)\n if ctx.db.position().entity_id().find(1).is_some() {\n ctx.db.position().entity_id().delete(&1);\n }\n ctx.db.position().insert(Position { entity_id: 1, x: 0, y: 0 });\n if ctx.db.velocity().entity_id().find(1).is_some() {\n ctx.db.velocity().entity_id().delete(&1);\n }\n ctx.db.velocity().insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n\n // Entity 2: pos=(10,0), vel=(-2,3)\n if ctx.db.position().entity_id().find(2).is_some() {\n ctx.db.position().entity_id().delete(&2);\n }\n ctx.db.position().insert(Position { entity_id: 2, x: 10, y: 0 });\n if ctx.db.velocity().entity_id().find(2).is_some() {\n ctx.db.velocity().entity_id().delete(&2);\n }\n ctx.db.velocity().insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n\n // Clean any existing next_position rows for these entities\n for &id in &[1, 2] {\n if ctx.db.next_position().entity_id().find(id).is_some() {\n ctx.db.next_position().entity_id().delete(&id);\n }\n }\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n let next = NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n };\n\n if ctx.db.next_position().entity_id().find(pos.entity_id).is_some() {\n let _ = ctx.db.next_position().entity_id().update(next);\n } else {\n let _ = ctx.db.next_position().insert(next);\n }\n }\n }\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.entity().insert(Entity { id: 1 });\n ctx.db.position().insert(Position { entity_id: 1, x: 0, y: 0 });\n ctx.db.velocity().insert(Velocity { entity_id: 1, vx: 1, vy: 0 });\n\n ctx.db.entity().insert(Entity { id: 2 });\n ctx.db.position().insert(Position { entity_id: 2, x: 10, y: 0 });\n ctx.db.velocity().insert(Velocity { entity_id: 2, vx: -2, vy: 3 });\n}\n\n#[reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n\n if let Some(mut existing) = ctx.db.next_position().entity_id().find(pos.entity_id) {\n existing.x = nx;\n existing.y = ny;\n let _ = ctx.db.next_position().entity_id().update(existing);\n } else {\n let _ = ctx.db.next_position().insert(NextPosition {\n entity_id: pos.entity_id,\n x: nx,\n y: ny,\n });\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-020-ecs-golden", @@ -1040,15 +1023,15 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41115/v1/database/c20022d8f2e2d547824e6b1c08b40372fd94143f227cebc3dd95d8debbce411d/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37093/v1/database/c200b735483077302c2220af9b35c9c621fc59499f0c0c65b07c0fa34aad4735/sql)\n", "phase": "sql" } }, - "ecs_seed_positions_count": { + "ecs_next_pos_entity2": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41115/v1/database/c20022d8f2e2d547824e6b1c08b40372fd94143f227cebc3dd95d8debbce411d/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37093/v1/database/c200b735483077302c2220af9b35c9c621fc59499f0c0c65b07c0fa34aad4735/sql)\n", "phase": "sql" } }, @@ -1060,31 +1043,31 @@ "llm_db": "schema-t-020-ecs-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41115", + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } }, - "ecs_step_next_positions_count": { + "ecs_seed_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41115/v1/database/c20022d8f2e2d547824e6b1c08b40372fd94143f227cebc3dd95d8debbce411d/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37093/v1/database/c200b735483077302c2220af9b35c9c621fc59499f0c0c65b07c0fa34aad4735/sql)\n", "phase": "sql" } }, - "ecs_next_pos_entity2": { + "ecs_step_next_positions_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41115/v1/database/c20022d8f2e2d547824e6b1c08b40372fd94143f227cebc3dd95d8debbce411d/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:37093/v1/database/c200b735483077302c2220af9b35c9c621fc59499f0c0c65b07c0fa34aad4735/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:07:57.743495455Z", - "finished_at": "2026-01-23T20:09:14.983916389Z" + "started_at": "2026-01-23T23:54:43.759002021Z", + "finished_at": "2026-01-23T23:55:24.734536804Z" }, "t_021_multi_column_index": { "hash": "da32ac1e4a22e29f27c1c75b3ce526f5ba909c184b9057d83384313a5e7b881a", @@ -1094,7 +1077,7 @@ "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[reducer]\npub fn seed(ctx: &ReducerContext) {\n let t = ctx.db.log();\n t.insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n t.insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n t.insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", + "llm_output": "use spacetimedb::{table, reducer, ReducerContext, Table};\n\n#[table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[reducer]\nfn seed(ctx: &ReducerContext) {\n let table = ctx.db.log();\n let _ = table.try_insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".to_string() });\n let _ = table.try_insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".to_string() });\n let _ = table.try_insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".to_string() });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -1102,13 +1085,13 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/rust/server/gpt-5/llm", "scorer_details": { - "mcindex_lookup_u7_d1": { + "mcindex_seed_count": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM log" } }, "schema_parity": { @@ -1119,7 +1102,7 @@ "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41115", + "server": "http://127.0.0.1:37093", "tables_diff": null, "tables_equal": true } @@ -1133,19 +1116,19 @@ "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" } }, - "mcindex_seed_count": { + "mcindex_lookup_u7_d1": { "pass": true, "partial": 1.0, "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM log" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:07:57.742772776Z", - "finished_at": "2026-01-23T20:08:35.084850930Z" + "started_at": "2026-01-23T23:54:44.022466387Z", + "finished_at": "2026-01-23T23:55:19.787278738Z" } } } @@ -1153,21 +1136,21 @@ }, { "mode": "docs", - "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "models": [ { "name": "GPT-5", "route_api_model": "gpt-5", "tasks": { "t_000_empty_reducers": { - "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_000_empty_reducers", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &spacetimedb::ReducerContext) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &spacetimedb::ReducerContext, _count: i32) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &spacetimedb::ReducerContext, _name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &spacetimedb::ReducerContext, _count: i32, _name: String) {\n}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &spacetimedb::ReducerContext, _active: bool, _ratio: f32, _label: String) {\n}", + "llm_output": "use spacetimedb::ReducerContext;\n\n#[spacetimedb::reducer]\npub fn empty_reducer_no_args(_ctx: &ReducerContext) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_int(_ctx: &ReducerContext, _count: i32) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_string(_ctx: &ReducerContext, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_two_args(_ctx: &ReducerContext, _count: i32, _name: String) {}\n\n#[spacetimedb::reducer]\npub fn empty_reducer_with_three_args(_ctx: &ReducerContext, _active: bool, _ratio: f32, _label: String) {}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-000-empty-reducers-golden", @@ -1185,18 +1168,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T20:12:10.018237719Z", - "finished_at": "2026-01-23T20:12:41.638729168Z" + "started_at": "2026-01-23T23:56:06.352823477Z", + "finished_at": "2026-01-23T23:57:14.696827098Z" }, "t_001_basic_tables": { - "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_001_basic_tables", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", + "llm_output": "use spacetimedb::Table;\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = product)]\npub struct Product {\n #[primary_key]\n id: i32,\n title: String,\n price: f32,\n in_stock: bool,\n}\n\n#[spacetimedb::table(name = note)]\npub struct Note {\n #[primary_key]\n id: i32,\n body: String,\n rating: i64,\n pinned: bool,\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-001-basic-tables-golden", @@ -1214,47 +1197,47 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T20:12:46.725514922Z", - "finished_at": "2026-01-23T20:13:51.799945305Z" + "started_at": "2026-01-23T23:56:06.340775944Z", + "finished_at": "2026-01-23T23:57:08.995142469Z" }, "t_002_scheduled_table": { - "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_002_scheduled_table", "lang": "rust", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use std::time::Duration;\nuse spacetimedb::{ReducerContext, ScheduleAt, Table};\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) {\n // No-op scheduled tick.\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.tick_timer().insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: ScheduleAt::Interval(Duration::from_micros(50_000).into()),\n });\n Ok(())\n}", + "llm_output": null, "category": "basics", "route_api_model": "gpt-5", - "golden_db": "basics-t-002-scheduled-table-golden", - "llm_db": "basics-t-002-scheduled-table-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_002_scheduled_table/rust/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_002_scheduled_table/rust/server/gpt-5/llm", + "golden_db": null, + "llm_db": null, + "work_dir_golden": null, + "work_dir_llm": null, "scorer_details": { - "schema_parity": { + "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-002-scheduled-table-golden`.\n", - "phase": "describe_golden" + "error": "LLM call timed out", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:13:28.197750258Z", - "finished_at": "2026-01-23T20:14:26.307958881Z" + "started_at": "2026-01-23T23:58:25.923588482Z", + "finished_at": "2026-01-23T23:58:25.923588482Z" }, "t_003_struct_in_table": { - "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_003_struct_in_table", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::SpacetimeType;\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n pos: Position,\n}", + "llm_output": "use spacetimedb::SpacetimeType;\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n pub x: i32,\n pub y: i32,\n}\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n pub id: i32,\n pub pos: Position,\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-003-struct-in-table-golden", @@ -1272,11 +1255,11 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T20:12:41.638758353Z", - "finished_at": "2026-01-23T20:13:47.479970679Z" + "started_at": "2026-01-23T23:56:06.343277423Z", + "finished_at": "2026-01-23T23:56:45.192100606Z" }, "t_004_insert": { - "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_004_insert", "lang": "rust", "golden_published": true, @@ -1309,18 +1292,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T20:12:46.734575843Z", - "finished_at": "2026-01-23T20:13:46.613223334Z" + "started_at": "2026-01-23T23:56:06.349640021Z", + "finished_at": "2026-01-23T23:56:49.150223443Z" }, "t_005_update": { - "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_005_update", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) {\n if ctx.db.user().id().find(id).is_some() {\n ctx.db.user().id().update(User { id, name, age, active });\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user, public)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn update_user(ctx: &ReducerContext, id: i32, name: String, age: i32, active: bool) -> Result<(), String> {\n if ctx.db.user().id().find(id).is_some() {\n ctx.db.user().id().update(User { id, name, age, active });\n Ok(())\n } else {\n Err(\"User not found\".to_string())\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -1355,11 +1338,11 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T20:12:10.015338220Z", - "finished_at": "2026-01-23T20:13:14.713691221Z" + "started_at": "2026-01-23T23:56:36.268165981Z", + "finished_at": "2026-01-23T23:57:06.114000259Z" }, "t_006_delete": { - "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_006_delete", "lang": "rust", "golden_published": true, @@ -1382,15 +1365,6 @@ "phase": "describe_golden" } }, - "delete_user_count_zero": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" - } - }, "seed_users_row": { "pass": false, "partial": 0.0, @@ -1399,76 +1373,59 @@ "phase": "sql_golden", "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" } + }, + "delete_user_count_zero": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 0, + "expected": 0, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1" + } } }, "vendor": "openai", - "started_at": "2026-01-23T20:12:09.994528341Z", - "finished_at": "2026-01-23T20:12:46.205776277Z" + "started_at": "2026-01-23T23:57:06.114026509Z", + "finished_at": "2026-01-23T23:57:33.036931801Z" }, "t_007_crud": { - "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_007_crud", "lang": "rust", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", - "total_tests": 4, - "passed_tests": 1, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer]\npub fn crud(ctx: &ReducerContext) {\n // insert (id=1, name=\"Alice\", age=30, active=true)\n ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".to_string(),\n age: 30,\n active: true,\n });\n\n // insert (id=2, name=\"Bob\", age=22, active=false)\n ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".to_string(),\n age: 22,\n active: false,\n });\n\n // update (id=1, name=\"Alice2\", age=31, active=false)\n if let Some(mut u1) = ctx.db.user().id().find(1) {\n u1.name = \"Alice2\".to_string();\n u1.age = 31;\n u1.active = false;\n ctx.db.user().id().update(u1);\n }\n\n // delete id=2\n ctx.db.user().id().delete(2);\n}", + "total_tests": 1, + "passed_tests": 0, + "llm_output": null, "category": "basics", "route_api_model": "gpt-5", - "golden_db": "basics-t-007-crud-golden", - "llm_db": "basics-t-007-crud-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_007_crud/rust/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_007_crud/rust/server/gpt-5/llm", + "golden_db": null, + "llm_db": null, + "work_dir_golden": null, + "work_dir_llm": null, "scorer_details": { - "crud_row_id1_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", - "phase": "call_reducer_golden" - } - }, - "crud_row_id2_deleted": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 0, - "expected": 0, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2" - } - }, - "crud_total_count_one": { - "pass": false, - "partial": 0.0, - "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user" - } - }, - "schema_parity": { + "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-007-crud-golden`.\n", - "phase": "describe_golden" + "error": "LLM call timed out", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:12:10.004797508Z", - "finished_at": "2026-01-23T20:13:37.162002816Z" + "started_at": "2026-01-23T23:58:25.923597248Z", + "finished_at": "2026-01-23T23:58:25.923597248Z" }, "t_008_index_lookup": { - "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_008_index_lookup", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) -> Result<(), String> {\n if let Some(user) = ctx.db.user().id().find(id) {\n ctx.db\n .result()\n .try_insert(ResultRow { id: user.id, name: user.name.clone() })?;\n Ok(())\n } else {\n Err(\"User not found\".to_string())\n }\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn lookup_user_name(ctx: &ReducerContext, id: i32) {\n if let Some(user) = ctx.db.user().id().find(id) {\n ctx.db.result().insert(ResultRow { id: user.id, name: user.name });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -1476,15 +1433,6 @@ "work_dir_golden": "target/llm-runs/basics/t_008_index_lookup/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_008_index_lookup/rust/server/gpt-5/llm", "scorer_details": { - "seed_user_row": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n", - "phase": "sql_golden", - "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" - } - }, "schema_parity": { "pass": false, "partial": 0.0, @@ -1500,21 +1448,30 @@ "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n", "phase": "call_reducer_golden" } + }, + "seed_user_row": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-008-index-lookup-golden`.\n", + "phase": "sql_golden", + "sql": "INSERT INTO user(id, name, age, active) VALUES (1, 'Alice', 30, true)" + } } }, "vendor": "openai", - "started_at": "2026-01-23T20:12:46.729438058Z", - "finished_at": "2026-01-23T20:13:54.235945228Z" + "started_at": "2026-01-23T23:56:49.150284606Z", + "finished_at": "2026-01-23T23:57:17.142408609Z" }, "t_009_init": { - "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_009_init", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 3, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.user().insert(User { id: 1, name: \"Alice\".to_string(), age: 30, active: true });\n ctx.db.user().insert(User { id: 2, name: \"Bob\".to_string(), age: 22, active: false });\n Ok(())\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n id: i32,\n name: String,\n age: i32,\n active: bool,\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.user().insert(User {\n id: 1,\n name: \"Alice\".into(),\n age: 30,\n active: true,\n });\n ctx.db.user().insert(User {\n id: 2,\n name: \"Bob\".into(),\n age: 22,\n active: false,\n });\n Ok(())\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -1522,22 +1479,31 @@ "work_dir_golden": "target/llm-runs/basics/t_009_init/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_009_init/rust/server/gpt-5/llm", "scorer_details": { - "init_total_two": { + "init_seed_alice": { "pass": true, "partial": 1.0, "notes": { - "actual": 2, - "expected": 2, - "sql": "SELECT COUNT(*) AS n FROM user" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" } }, - "init_seed_alice": { + "init_seed_bob": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=1 AND name='Alice' AND age=30 AND active=true" + "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" + } + }, + "init_total_two": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 2, + "expected": 2, + "sql": "SELECT COUNT(*) AS n FROM user" } }, "schema_parity": { @@ -1547,30 +1513,21 @@ "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-009-init-golden`.\n", "phase": "describe_golden" } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM user WHERE id=2 AND name='Bob' AND age=22 AND active=false" - } } }, "vendor": "openai", - "started_at": "2026-01-23T20:12:10.009828612Z", - "finished_at": "2026-01-23T20:12:45.617807272Z" + "started_at": "2026-01-23T23:56:45.192125865Z", + "finished_at": "2026-01-23T23:57:10.060687704Z" }, "t_010_connect": { - "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_010_connect", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = event)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event { id: 0, kind: \"connected\".into() });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event { id: 0, kind: \"disconnected\".into() });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = event)]\npub struct Event {\n #[primary_key]\n #[auto_inc]\n id: i32,\n kind: String,\n}\n\n#[spacetimedb::reducer(client_connected)]\npub fn client_connected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event { id: 0, kind: \"connected\".to_string() });\n}\n\n#[spacetimedb::reducer(client_disconnected)]\npub fn client_disconnected(ctx: &ReducerContext) {\n ctx.db.event().insert(Event { id: 0, kind: \"disconnected\".to_string() });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-010-connect-golden", @@ -1588,18 +1545,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T20:12:09.998960857Z", - "finished_at": "2026-01-23T20:13:28.197638116Z" + "started_at": "2026-01-23T23:56:06.359406815Z", + "finished_at": "2026-01-23T23:56:36.268095898Z" }, "t_011_helper_function": { - "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_011_helper_function", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[spacetimedb::reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n let s = add(a, b);\n ctx.db.result().insert(ResultRow { id, sum: s });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n sum: i32,\n}\n\nfn add(a: i32, b: i32) -> i32 {\n a + b\n}\n\n#[spacetimedb::reducer]\npub fn compute_sum(ctx: &ReducerContext, id: i32, a: i32, b: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n sum: add(a, b),\n });\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-011-helper-function-golden", @@ -1607,14 +1564,6 @@ "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/rust/server/golden", "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/rust/server/gpt-5/llm", "scorer_details": { - "helper_func_sum_parity": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-011-helper-function-golden`.\n", - "phase": "call_reducer_golden" - } - }, "schema_parity": { "pass": false, "partial": 0.0, @@ -1631,21 +1580,29 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1 AND sum=5" } + }, + "helper_func_sum_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `basics-t-011-helper-function-golden`.\n", + "phase": "call_reducer_golden" + } } }, "vendor": "openai", - "started_at": "2026-01-23T20:13:14.723917629Z", - "finished_at": "2026-01-23T20:14:00.225654958Z" + "started_at": "2026-01-23T23:56:06.346316813Z", + "finished_at": "2026-01-23T23:57:35.149751175Z" }, "t_012_spacetime_product_type": { - "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_012_spacetime_product_type", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[derive(spacetimedb::SpacetimeType, Clone, Debug)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType)]\npub struct Score {\n left: i32,\n right: i32,\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Score,\n}\n\n#[spacetimedb::reducer]\npub fn set_score(ctx: &ReducerContext, id: i32, left: i32, right: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Score { left, right },\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-012-spacetime-product-type-golden", @@ -1680,47 +1637,64 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T20:12:09.991277650Z", - "finished_at": "2026-01-23T20:12:46.725468970Z" + "started_at": "2026-01-23T23:57:35.149826204Z", + "finished_at": "2026-01-23T23:57:59.916792151Z" }, "t_013_spacetime_sum_type": { - "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_013_spacetime_sum_type", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, + "total_tests": 3, "passed_tests": 0, - "llm_output": null, + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = result)]\npub struct ResultRow {\n #[primary_key]\n id: i32,\n value: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn set_circle(ctx: &ReducerContext, id: i32, radius: i32) {\n ctx.db.result().insert(ResultRow {\n id,\n value: Shape::Circle(radius),\n });\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-013-spacetime-sum-type-golden", + "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { + "sum_type_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "LLM call timed out", - "phase": "build_or_publish" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-013-spacetime-sum-type-golden`.\n", + "phase": "call_reducer_golden" + } + }, + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-013-spacetime-sum-type-golden`.\n", + "phase": "describe_golden" + } + }, + "sum_type_row_count": { + "pass": false, + "partial": 0.0, + "notes": { + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM result WHERE id=1" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:14:26.308049242Z", - "finished_at": "2026-01-23T20:14:26.308049242Z" + "started_at": "2026-01-23T23:57:36.363684916Z", + "finished_at": "2026-01-23T23:57:57.614971862Z" }, "t_014_elementary_columns": { - "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_014_elementary_columns", "lang": "rust", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 0, - "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5_f32,\n ratio: 2.25,\n active: true,\n name: \"Alice\".into(),\n });\n}", + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = primitive)]\npub struct Primitive {\n #[primary_key]\n id: i32,\n count: i32,\n total: i64,\n price: f32,\n ratio: f64,\n active: bool,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.primitive().insert(Primitive {\n id: 1,\n count: 2,\n total: 3_000_000_000,\n price: 1.5,\n ratio: 2.25,\n active: true,\n name: \"Alice\".into(),\n });\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-014-elementary-columns-golden", @@ -1728,13 +1702,12 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/rust/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/rust/server/gpt-5/llm", "scorer_details": { - "elementary_columns_row_count": { + "elementary_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "actual": 0, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", + "phase": "call_reducer_golden" } }, "schema_parity": { @@ -1745,221 +1718,376 @@ "phase": "describe_golden" } }, - "elementary_columns_row_parity": { + "elementary_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-014-elementary-columns-golden`.\n", - "phase": "call_reducer_golden" + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM primitive WHERE id=1" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:12:09.987997099Z", - "finished_at": "2026-01-23T20:13:50.463947582Z" + "started_at": "2026-01-23T23:57:36.357778606Z", + "finished_at": "2026-01-23T23:57:59.614595614Z" }, "t_015_product_type_columns": { - "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_015_product_type_columns", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, + "total_tests": 3, "passed_tests": 0, - "llm_output": null, + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Address {\n street: String,\n zip: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Position {\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = profile)]\npub struct Profile {\n #[primary_key]\n id: i32,\n home: Address,\n work: Address,\n pos: Position,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for p in ctx.db.profile().iter() {\n ctx.db.profile().id().delete(&p.id);\n }\n ctx.db.profile().insert(Profile {\n id: 1,\n home: Address {\n street: \"1 Main\".to_string(),\n zip: 11111,\n },\n work: Address {\n street: \"2 Broad\".to_string(),\n zip: 22222,\n },\n pos: Position { x: 7, y: 9 },\n });\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-015-product-type-columns-golden", + "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_015_product_type_columns/rust/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_015_product_type_columns/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { + "product_type_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "LLM call timed out", - "phase": "build_or_publish" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", + "phase": "call_reducer_golden" + } + }, + "product_type_columns_row_count": { + "pass": false, + "partial": 0.0, + "notes": { + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM profile WHERE id=1" + } + }, + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-015-product-type-columns-golden`.\n", + "phase": "describe_golden" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:14:26.308063959Z", - "finished_at": "2026-01-23T20:14:26.308063959Z" + "started_at": "2026-01-23T23:57:40.591120346Z", + "finished_at": "2026-01-23T23:58:19.356995796Z" }, "t_016_sum_type_columns": { - "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_016_sum_type_columns", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, + "total_tests": 3, "passed_tests": 0, - "llm_output": null, + "llm_output": "use spacetimedb::{ReducerContext, SpacetimeType, Table};\n\n#[derive(SpacetimeType, Clone, Debug)]\npub struct Rect {\n width: i32,\n height: i32,\n}\n\n#[derive(SpacetimeType, Clone, Debug)]\npub enum Shape {\n Circle(i32),\n Rectangle(Rect),\n}\n\n#[spacetimedb::table(name = drawing)]\npub struct Drawing {\n #[primary_key]\n id: i32,\n a: Shape,\n b: Shape,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n for row in ctx.db.drawing().iter() {\n ctx.db.drawing().id().delete(&row.id);\n }\n ctx.db.drawing().insert(Drawing {\n id: 1,\n a: Shape::Circle(10),\n b: Shape::Rectangle(Rect { width: 4, height: 6 }),\n });\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-016-sum-type-columns-golden", + "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_016_sum_type_columns/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { + "sum_type_columns_row_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "LLM call timed out", - "phase": "build_or_publish" + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", + "phase": "call_reducer_golden" + } + }, + "sum_type_columns_row_count": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40199/v1/database/c200b6269c628fcac98f15b7068e57501de59c811b92dafc5267eddaca62177b/sql)\n", + "phase": "sql" + } + }, + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-016-sum-type-columns-golden`.\n", + "phase": "describe_golden" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:14:26.308044663Z", - "finished_at": "2026-01-23T20:14:26.308044663Z" + "started_at": "2026-01-23T23:57:48.299178998Z", + "finished_at": "2026-01-23T23:58:25.922826951Z" }, "t_017_scheduled_columns": { - "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_017_scheduled_columns", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 2, + "passed_tests": 1, + "llm_output": "use spacetimedb::{ReducerContext, ScheduleAt, Table};\nuse std::time::Duration;\n\n#[spacetimedb::table(name = tick_timer, scheduled(tick))]\npub struct TickTimer {\n #[primary_key]\n #[auto_inc]\n scheduled_id: u64,\n scheduled_at: ScheduleAt,\n}\n\n#[spacetimedb::reducer]\npub fn tick(_ctx: &ReducerContext, _timer: TickTimer) -> Result<(), String> {\n Ok(())\n}\n\n#[spacetimedb::reducer(init)]\npub fn init(ctx: &ReducerContext) -> Result<(), String> {\n let interval = ScheduleAt::Interval(Duration::from_micros(50_000).into());\n ctx.db.tick_timer().try_insert(TickTimer {\n scheduled_id: 0,\n scheduled_at: interval,\n })?;\n Ok(())\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-017-scheduled-columns-golden", + "llm_db": "schema-t-017-scheduled-columns-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_017_scheduled_columns/rust/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_017_scheduled_columns/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "LLM call timed out", - "phase": "build_or_publish" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-017-scheduled-columns-golden`.\n", + "phase": "describe_golden" + } + }, + "scheduled_seeded_one_row": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM tick_timer WHERE scheduled_id>=0" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:14:26.308033084Z", - "finished_at": "2026-01-23T20:14:26.308033084Z" + "started_at": "2026-01-23T23:57:33.036962112Z", + "finished_at": "2026-01-23T23:58:11.840818516Z" }, "t_018_constraints": { - "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_018_constraints", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, + "total_tests": 3, "passed_tests": 0, - "llm_output": null, + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = account, index(name = by_name, btree(columns = [name])))]\npub struct Account {\n #[primary_key]\n id: i32,\n #[unique]\n email: String,\n name: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) -> Result<(), String> {\n ctx.db.account().try_insert(Account {\n id: 1,\n email: \"a@example.com\".into(),\n name: \"Alice\".into(),\n })?;\n ctx.db.account().try_insert(Account {\n id: 2,\n email: \"b@example.com\".into(),\n name: \"Bob\".into(),\n })?;\n Ok(())\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-018-constraints-golden", + "llm_db": "schema-t-018-constraints-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_018_constraints/rust/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_018_constraints/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { + "constraints_seed_two_rows": { "pass": false, "partial": 0.0, "notes": { - "error": "LLM call timed out", - "phase": "build_or_publish" + "actual": 0, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM account WHERE id=2" + } + }, + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-018-constraints-golden`.\n", + "phase": "describe_golden" + } + }, + "constraints_row_parity_after_seed": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime call failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-018-constraints-golden`.\n", + "phase": "call_reducer_golden" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:14:26.308021447Z", - "finished_at": "2026-01-23T20:14:26.308021447Z" + "started_at": "2026-01-23T23:57:14.696906982Z", + "finished_at": "2026-01-23T23:57:40.591097329Z" }, "t_019_many_to_many": { - "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_019_many_to_many", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 5, + "passed_tests": 4, + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = user)]\npub struct User {\n #[primary_key]\n user_id: i32,\n name: String,\n}\n\n#[spacetimedb::table(name = group)]\npub struct Group {\n #[primary_key]\n group_id: i32,\n title: String,\n}\n\n#[spacetimedb::table(\n name = membership,\n index(name = by_user, btree(columns = [user_id])),\n index(name = by_group, btree(columns = [group_id]))\n)]\npub struct Membership {\n #[primary_key]\n id: i32,\n user_id: i32,\n group_id: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n ctx.db.user().insert(User { user_id: 1, name: \"Alice\".to_string() });\n ctx.db.user().insert(User { user_id: 2, name: \"Bob\".to_string() });\n\n ctx.db.group().insert(Group { group_id: 10, title: \"Admin\".to_string() });\n ctx.db.group().insert(Group { group_id: 20, title: \"Dev\".to_string() });\n\n ctx.db.membership().insert(Membership { id: 1, user_id: 1, group_id: 10 });\n ctx.db.membership().insert(Membership { id: 2, user_id: 1, group_id: 20 });\n ctx.db.membership().insert(Membership { id: 3, user_id: 2, group_id: 20 });\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-019-many-to-many-golden", + "llm_db": "schema-t-019-many-to-many-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/rust/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { + "memberships_three_rows": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM membership" + } + }, + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "LLM call timed out", - "phase": "build_or_publish" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-019-many-to-many-golden`.\n", + "phase": "describe_golden" + } + }, + "m2m_has_1_10": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=10" + } + }, + "m2m_has_1_20": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=1 AND group_id=20" + } + }, + "m2m_has_2_20": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM membership WHERE user_id=2 AND group_id=20" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:14:26.308054994Z", - "finished_at": "2026-01-23T20:14:26.308054994Z" + "started_at": "2026-01-23T23:57:17.142551153Z", + "finished_at": "2026-01-23T23:57:49.896790577Z" }, "t_020_ecs": { - "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_020_ecs", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, + "total_tests": 5, "passed_tests": 0, - "llm_output": null, + "llm_output": "use spacetimedb::{ReducerContext, Table};\n\n#[spacetimedb::table(name = entity)]\npub struct Entity {\n #[primary_key]\n id: i32,\n}\n\n#[spacetimedb::table(name = position)]\npub struct Position {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::table(name = velocity)]\npub struct Velocity {\n #[primary_key]\n entity_id: i32,\n vx: i32,\n vy: i32,\n}\n\n#[spacetimedb::table(name = next_position)]\npub struct NextPosition {\n #[primary_key]\n entity_id: i32,\n x: i32,\n y: i32,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &ReducerContext) {\n // entity 1\n let e1 = 1;\n if ctx.db.entity().id().find(e1).is_none() {\n ctx.db.entity().insert(Entity { id: e1 });\n }\n let p1 = Position { entity_id: e1, x: 0, y: 0 };\n if ctx.db.position().entity_id().find(e1).is_some() {\n ctx.db.position().entity_id().update(p1);\n } else {\n ctx.db.position().insert(p1);\n }\n let v1 = Velocity { entity_id: e1, vx: 1, vy: 0 };\n if ctx.db.velocity().entity_id().find(e1).is_some() {\n ctx.db.velocity().entity_id().update(v1);\n } else {\n ctx.db.velocity().insert(v1);\n }\n\n // entity 2\n let e2 = 2;\n if ctx.db.entity().id().find(e2).is_none() {\n ctx.db.entity().insert(Entity { id: e2 });\n }\n let p2 = Position { entity_id: e2, x: 10, y: 0 };\n if ctx.db.position().entity_id().find(e2).is_some() {\n ctx.db.position().entity_id().update(p2);\n } else {\n ctx.db.position().insert(p2);\n }\n let v2 = Velocity { entity_id: e2, vx: -2, vy: 3 };\n if ctx.db.velocity().entity_id().find(e2).is_some() {\n ctx.db.velocity().entity_id().update(v2);\n } else {\n ctx.db.velocity().insert(v2);\n }\n}\n\n#[spacetimedb::reducer]\npub fn step(ctx: &ReducerContext) {\n for pos in ctx.db.position().iter() {\n if let Some(vel) = ctx.db.velocity().entity_id().find(pos.entity_id) {\n let nx = pos.x + vel.vx;\n let ny = pos.y + vel.vy;\n let np = NextPosition { entity_id: pos.entity_id, x: nx, y: ny };\n if ctx.db.next_position().entity_id().find(pos.entity_id).is_some() {\n ctx.db.next_position().entity_id().update(np);\n } else {\n ctx.db.next_position().insert(np);\n }\n }\n }\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-020-ecs-golden", + "llm_db": "schema-t-020-ecs-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_020_ecs/rust/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_020_ecs/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { + "ecs_next_pos_entity1": { "pass": false, "partial": 0.0, "notes": { - "error": "LLM call timed out", - "phase": "build_or_publish" + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40199/v1/database/c200a61076d09b2b7988361f7eb75cbbe85ce3a70a733b7cf52469970ead2ae5/sql)\n", + "phase": "sql" + } + }, + "schema_parity": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-020-ecs-golden`.\n", + "phase": "describe_golden" + } + }, + "ecs_next_pos_entity2": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40199/v1/database/c200a61076d09b2b7988361f7eb75cbbe85ce3a70a733b7cf52469970ead2ae5/sql)\n", + "phase": "sql" + } + }, + "ecs_seed_positions_count": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40199/v1/database/c200a61076d09b2b7988361f7eb75cbbe85ce3a70a733b7cf52469970ead2ae5/sql)\n", + "phase": "sql" + } + }, + "ecs_step_next_positions_count": { + "pass": false, + "partial": 0.0, + "notes": { + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:40199/v1/database/c200a61076d09b2b7988361f7eb75cbbe85ce3a70a733b7cf52469970ead2ae5/sql)\n", + "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:14:26.308059854Z", - "finished_at": "2026-01-23T20:14:26.308059854Z" + "started_at": "2026-01-23T23:57:08.995173371Z", + "finished_at": "2026-01-23T23:57:51.538787094Z" }, "t_021_multi_column_index": { - "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "task": "t_021_multi_column_index", "lang": "rust", - "golden_published": false, + "golden_published": true, "model_name": "GPT-5", - "total_tests": 1, - "passed_tests": 0, - "llm_output": null, + "total_tests": 4, + "passed_tests": 3, + "llm_output": "use spacetimedb::Table;\n\n#[spacetimedb::table(name = log, index(name = by_user_day, btree(columns = [user_id, day])))]\npub struct Log {\n #[primary_key]\n id: i32,\n user_id: i32,\n day: i32,\n message: String,\n}\n\n#[spacetimedb::reducer]\npub fn seed(ctx: &spacetimedb::ReducerContext) {\n let ids: Vec = ctx.db.log().iter().map(|r| r.id).collect();\n for id in ids {\n ctx.db.log().id().delete(id);\n }\n ctx.db.log().insert(Log { id: 1, user_id: 7, day: 1, message: \"a\".into() });\n ctx.db.log().insert(Log { id: 2, user_id: 7, day: 2, message: \"b\".into() });\n ctx.db.log().insert(Log { id: 3, user_id: 9, day: 1, message: \"c\".into() });\n}", "category": "schema", "route_api_model": "gpt-5", - "golden_db": null, - "llm_db": null, - "work_dir_golden": null, - "work_dir_llm": null, + "golden_db": "schema-t-021-multi-column-index-golden", + "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", + "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/rust/server/golden", + "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/rust/server/gpt-5/llm", "scorer_details": { - "publish_error": { + "mcindex_lookup_u7_d2": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=2" + } + }, + "mcindex_lookup_u7_d1": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM log WHERE user_id=7 AND day=1" + } + }, + "schema_parity": { "pass": false, "partial": 0.0, "notes": { - "error": "LLM call timed out", - "phase": "build_or_publish" + "error": "describe failed: WARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: failed to find database `schema-t-021-multi-column-index-golden`.\n", + "phase": "describe_golden" + } + }, + "mcindex_seed_count": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM log" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:14:26.308040547Z", - "finished_at": "2026-01-23T20:14:26.308040547Z" + "started_at": "2026-01-23T23:57:10.060719477Z", + "finished_at": "2026-01-23T23:57:48.299152677Z" } } } @@ -2150,14 +2278,14 @@ "modes": [ { "mode": "docs", - "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "models": [ { "name": "GPT-5", "route_api_model": "gpt-5", "tasks": { "t_000_empty_reducers": { - "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_000_empty_reducers", "lang": "csharp", "golden_published": true, @@ -2180,25 +2308,25 @@ "llm_db": "basics-t-000-empty-reducers-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41045", + "server": "http://127.0.0.1:41165", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:20:05.908071Z", - "finished_at": "2026-01-23T20:21:02.544054377Z" + "started_at": "2026-01-24T00:00:26.239129005Z", + "finished_at": "2026-01-24T00:01:25.207476766Z" }, "t_001_basic_tables": { - "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_001_basic_tables", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 1, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Product\", Public = true)]\n public partial struct Product\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table(Name = \"Note\", Public = true)]\n public partial struct Note\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Product\")]\n public partial struct Product\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Title;\n public float Price;\n public bool InStock;\n }\n\n [SpacetimeDB.Table(Name = \"Note\")]\n public partial struct Note\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Body;\n public long Rating;\n public bool Pinned;\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-001-basic-tables-golden", @@ -2214,52 +2342,47 @@ "llm_db": "basics-t-001-basic-tables-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41045", + "server": "http://127.0.0.1:41165", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:20:17.520008030Z", - "finished_at": "2026-01-23T20:21:03.652127542Z" + "started_at": "2026-01-23T23:59:46.761379627Z", + "finished_at": "2026-01-24T00:00:40.834487811Z" }, "t_002_scheduled_table": { - "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_002_scheduled_table", "lang": "csharp", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", "total_tests": 1, - "passed_tests": 1, - "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey, SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n}", + "passed_tests": 0, + "llm_output": null, "category": "basics", "route_api_model": "gpt-5", - "golden_db": "basics-t-002-scheduled-table-golden", - "llm_db": "basics-t-002-scheduled-table-gpt-5-llm", - "work_dir_golden": "target/llm-runs/basics/t_002_scheduled_table/csharp/server/golden", - "work_dir_llm": "target/llm-runs/basics/t_002_scheduled_table/csharp/server/gpt-5/llm", + "golden_db": null, + "llm_db": null, + "work_dir_golden": null, + "work_dir_llm": null, "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, + "publish_error": { + "pass": false, + "partial": 0.0, "notes": { - "golden_db": "basics-t-002-scheduled-table-golden", - "llm_db": "basics-t-002-scheduled-table-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41045", - "tables_diff": null, - "tables_equal": true + "error": "POST https://api.openai.com/v1/responses -> 500 Internal Server Error: {\n \"error\": {\n \"message\": \"An error occurred while processing your request. You can retry your request, or contact us through our help center at help.openai.com if the error persists. Please include the request ID req_52a83fd28c3442fd8fef03cd91bfc2b9 in your message.\",\n \"type\": \"server_error\",\n \"param\": null,\n \"code\": \"server_error\"\n }\n}", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:21:05.396968262Z", - "finished_at": "2026-01-23T20:21:59.590160864Z" + "started_at": "2026-01-24T00:05:38.729207961Z", + "finished_at": "2026-01-24T00:05:38.729207961Z" }, "t_003_struct_in_table": { - "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_003_struct_in_table", "lang": "csharp", "golden_published": true, @@ -2282,18 +2405,18 @@ "llm_db": "basics-t-003-struct-in-table-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41045", + "server": "http://127.0.0.1:41165", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:20:12.029574819Z", - "finished_at": "2026-01-23T20:20:58.944028610Z" + "started_at": "2026-01-23T23:59:46.763860791Z", + "finished_at": "2026-01-24T00:00:48.083440124Z" }, "t_004_insert": { - "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_004_insert", "lang": "csharp", "golden_published": true, @@ -2324,7 +2447,7 @@ "llm_out": "Id | Name | Age | Active ----+---------+-----+-------- 1 | \"Alice\" | 30 | true", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "InsertUser", - "server": "http://127.0.0.1:41045" + "server": "http://127.0.0.1:41165" } }, "schema_parity": { @@ -2335,25 +2458,25 @@ "llm_db": "basics-t-004-insert-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41045", + "server": "http://127.0.0.1:41165", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:21:02.544101204Z", - "finished_at": "2026-01-23T20:21:46.575966420Z" + "started_at": "2026-01-23T23:59:46.769869807Z", + "finished_at": "2026-01-24T00:00:35.226129696Z" }, "t_005_update": { - "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_005_update", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n ctx.Db.User.Id.Update(new User\n {\n Id = id,\n Name = name,\n Age = age,\n Active = active\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void UpdateUser(ReducerContext ctx, int id, string name, int age, bool active)\n {\n if (ctx.Db.User.Id.Find(id) is User user)\n {\n user.Id = id;\n user.Name = name;\n user.Age = age;\n user.Active = active;\n ctx.Db.User.Id.Update(user);\n }\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-005-update-golden", @@ -2361,13 +2484,6 @@ "work_dir_golden": "target/llm-runs/basics/t_005_update/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_005_update/csharp/server/gpt-5/llm", "scorer_details": { - "seed_users_row": { - "pass": true, - "partial": 1.0, - "notes": { - "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -2376,7 +2492,7 @@ "llm_db": "basics-t-005-update-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41045", + "server": "http://127.0.0.1:41165", "tables_diff": null, "tables_equal": true } @@ -2397,23 +2513,30 @@ "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", "reducer": "UpdateUser", - "server": "http://127.0.0.1:41045" + "server": "http://127.0.0.1:41165" + } + }, + "seed_users_row": { + "pass": true, + "partial": 1.0, + "notes": { + "sql": "INSERT INTO User(Id, Name, Age, Active) VALUES (1, 'Alice', 30, true)" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:19:53.867992725Z", - "finished_at": "2026-01-23T20:21:05.391950708Z" + "started_at": "2026-01-24T00:00:48.083499328Z", + "finished_at": "2026-01-24T00:02:11.608478542Z" }, "t_006_delete": { - "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_006_delete", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.User.Id.Delete(id);\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void DeleteUser(ReducerContext ctx, int id)\n {\n ctx.Db.User.Id.Delete(id);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-006-delete-golden", @@ -2438,7 +2561,7 @@ "llm_db": "basics-t-006-delete-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41045", + "server": "http://127.0.0.1:41165", "tables_diff": null, "tables_equal": true } @@ -2452,18 +2575,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T20:19:10.793697386Z", - "finished_at": "2026-01-23T20:19:53.867926470Z" + "started_at": "2026-01-24T00:02:06.759067811Z", + "finished_at": "2026-01-24T00:02:35.245051100Z" }, "t_007_crud": { - "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_007_crud", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n var found = ctx.Db.User.Id.Find(1);\n if (found is User user1)\n {\n user1.Name = \"Alice2\";\n user1.Age = 31;\n user1.Active = false;\n ctx.Db.User.Id.Update(user1);\n }\n\n ctx.Db.User.Id.Delete(2);\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\", Public = true)]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer]\n public static void Crud(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n\n if (ctx.Db.User.Id.Find(1) is User u1)\n {\n u1.Name = \"Alice2\";\n u1.Age = 31;\n u1.Active = false;\n ctx.Db.User.Id.Update(u1);\n }\n\n ctx.Db.User.Id.Delete(2);\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-007-crud-golden", @@ -2471,15 +2594,6 @@ "work_dir_golden": "target/llm-runs/basics/t_007_crud/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_007_crud/csharp/server/gpt-5/llm", "scorer_details": { - "crud_total_count_one": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User" - } - }, "crud_row_id2_deleted": { "pass": true, "partial": 1.0, @@ -2489,47 +2603,56 @@ "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2" } }, - "schema_parity": { + "crud_row_id1_parity": { "pass": true, "partial": 1.0, "notes": { + "args": [], "golden_db": "basics-t-007-crud-golden", + "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "llm_db": "basics-t-007-crud-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41045", - "tables_diff": null, - "tables_equal": true + "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", + "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", + "reducer": "Crud", + "server": "http://127.0.0.1:41165" } }, - "crud_row_id1_parity": { + "crud_total_count_one": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM User" + } + }, + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "args": [], "golden_db": "basics-t-007-crud-golden", - "golden_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", "llm_db": "basics-t-007-crud-gpt-5-llm", - "llm_out": "Id | Name | Age | Active ----+----------+-----+-------- 1 | \"Alice2\" | 31 | false", - "query": "SELECT Id, Name, Age, Active FROM User WHERE Id=1", - "reducer": "Crud", - "server": "http://127.0.0.1:41045" + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41165", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:19:29.508011285Z", - "finished_at": "2026-01-23T20:20:17.519940667Z" + "started_at": "2026-01-24T00:00:44.726730327Z", + "finished_at": "2026-01-24T00:02:06.759033129Z" }, "t_008_index_lookup": { - "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_008_index_lookup", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial class User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name = \"\";\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial class Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name = \"\";\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n if (ctx.Db.User.Id.Find(id) is User u)\n {\n ctx.Db.Result.Insert(new Result { Id = u.Id, Name = u.Name });\n }\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial class User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial class Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void LookupUserName(ReducerContext ctx, int id)\n {\n var user = ctx.Db.User.Id.Find(id);\n if (user != null)\n {\n ctx.Db.Result.Insert(new Result { Id = user.Id, Name = user.Name });\n }\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-008-index-lookup-golden", @@ -2545,7 +2668,7 @@ "llm_db": "basics-t-008-index-lookup-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41045", + "server": "http://127.0.0.1:41165", "tables_diff": null, "tables_equal": true } @@ -2570,23 +2693,23 @@ "llm_out": "Id | Name ----+--------- 1 | \"Alice\"", "query": "SELECT Id, Name FROM Result WHERE Id=1", "reducer": "LookupUserName", - "server": "http://127.0.0.1:41045" + "server": "http://127.0.0.1:41165" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:20:58.944081428Z", - "finished_at": "2026-01-23T20:21:57.704447793Z" + "started_at": "2026-01-24T00:01:34.491862134Z", + "finished_at": "2026-01-24T00:02:24.983219977Z" }, "t_009_init": { - "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_009_init", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(SpacetimeDB.ReducerKind.Init)]\n public static void Init(SpacetimeDB.ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"User\")]\n public partial struct User\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public string Name;\n public int Age;\n public bool Active;\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.User.Insert(new User { Id = 1, Name = \"Alice\", Age = 30, Active = true });\n ctx.Db.User.Insert(new User { Id = 2, Name = \"Bob\", Age = 22, Active = false });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-009-init-golden", @@ -2612,6 +2735,15 @@ "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=1 AND Name='Alice' AND Age=30 AND Active=true" } }, + "init_seed_bob": { + "pass": true, + "partial": 1.0, + "notes": { + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" + } + }, "schema_parity": { "pass": true, "partial": 1.0, @@ -2620,27 +2752,18 @@ "llm_db": "basics-t-009-init-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41045", + "server": "http://127.0.0.1:41165", "tables_diff": null, "tables_equal": true } - }, - "init_seed_bob": { - "pass": true, - "partial": 1.0, - "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM User WHERE Id=2 AND Name='Bob' AND Age=22 AND Active=false" - } } }, "vendor": "openai", - "started_at": "2026-01-23T20:19:36.893890047Z", - "finished_at": "2026-01-23T20:20:12.029514649Z" + "started_at": "2026-01-24T00:01:25.207506306Z", + "finished_at": "2026-01-24T00:02:23.314749141Z" }, "t_010_connect": { - "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_010_connect", "lang": "csharp", "golden_published": true, @@ -2663,25 +2786,25 @@ "llm_db": "basics-t-010-connect-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41045", + "server": "http://127.0.0.1:41165", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:19:23.574032624Z", - "finished_at": "2026-01-23T20:20:05.908027898Z" + "started_at": "2026-01-24T00:00:40.834521245Z", + "finished_at": "2026-01-24T00:01:34.491841870Z" }, "t_011_helper_function": { - "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_011_helper_function", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n public static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result { Id = id, Sum = Add(a, b) });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Result\", Public = true)]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Sum;\n }\n\n public static int Add(int a, int b)\n {\n return a + b;\n }\n\n [SpacetimeDB.Reducer]\n public static void ComputeSum(ReducerContext ctx, int id, int a, int b)\n {\n ctx.Db.Result.Insert(new Result\n {\n Id = id,\n Sum = Add(a, b)\n });\n }\n}", "category": "basics", "route_api_model": "gpt-5", "golden_db": "basics-t-011-helper-function-golden", @@ -2689,13 +2812,22 @@ "work_dir_golden": "target/llm-runs/basics/t_011_helper_function/csharp/server/golden", "work_dir_llm": "target/llm-runs/basics/t_011_helper_function/csharp/server/gpt-5/llm", "scorer_details": { - "helper_func_sum_abs": { + "helper_func_sum_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1 AND Sum=5" + "args": [ + 1, + 2, + 3 + ], + "golden_db": "basics-t-011-helper-function-golden", + "golden_out": "Id | Sum ----+----- 1 | 5", + "llm_db": "basics-t-011-helper-function-gpt-5-llm", + "llm_out": "Id | Sum ----+----- 1 | 5", + "query": "SELECT Id, Sum FROM Result WHERE Id=1", + "reducer": "ComputeSum", + "server": "http://127.0.0.1:41165" } }, "schema_parity": { @@ -2706,36 +2838,27 @@ "llm_db": "basics-t-011-helper-function-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41045", + "server": "http://127.0.0.1:41165", "tables_diff": null, "tables_equal": true } }, - "helper_func_sum_parity": { + "helper_func_sum_abs": { "pass": true, "partial": 1.0, "notes": { - "args": [ - 1, - 2, - 3 - ], - "golden_db": "basics-t-011-helper-function-golden", - "golden_out": "Id | Sum ----+----- 1 | 5", - "llm_db": "basics-t-011-helper-function-gpt-5-llm", - "llm_out": "Id | Sum ----+----- 1 | 5", - "query": "SELECT Id, Sum FROM Result WHERE Id=1", - "reducer": "ComputeSum", - "server": "http://127.0.0.1:41045" + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1 AND Sum=5" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:21:05.392014348Z", - "finished_at": "2026-01-23T20:21:51.189712882Z" + "started_at": "2026-01-23T23:59:46.766827274Z", + "finished_at": "2026-01-24T00:00:26.239091882Z" }, "t_012_spacetime_product_type": { - "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_012_spacetime_product_type", "lang": "csharp", "golden_published": true, @@ -2758,7 +2881,7 @@ "llm_db": "schema-t-012-spacetime-product-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41045", + "server": "http://127.0.0.1:41165", "tables_diff": null, "tables_equal": true } @@ -2778,7 +2901,7 @@ "llm_out": "Id | Value ----+----------------------- 1 | (Left = 2, Right = 3)", "query": "SELECT Id, Value FROM Result WHERE Id=1", "reducer": "SetScore", - "server": "http://127.0.0.1:41045" + "server": "http://127.0.0.1:41165" } }, "product_type_row_count": { @@ -2792,18 +2915,18 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T20:18:54.161733902Z", - "finished_at": "2026-01-23T20:19:36.893826573Z" + "started_at": "2026-01-24T00:03:16.504458119Z", + "finished_at": "2026-01-24T00:03:55.909124187Z" }, "t_013_spacetime_sum_type": { - "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_013_spacetime_sum_type", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 3, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n var value = new Shape.Circle(new Circle { Radius = radius });\n ctx.Db.Result.Insert(new Result { Id = id, Value = value });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Result\")]\n public partial struct Result\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape Value;\n }\n\n [SpacetimeDB.Reducer]\n public static void SetCircle(ReducerContext ctx, int id, int radius)\n {\n var shape = new Shape.Circle(new Circle { Radius = radius });\n ctx.Db.Result.Insert(new Result { Id = id, Value = shape });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-013-spacetime-sum-type-golden", @@ -2811,23 +2934,6 @@ "work_dir_golden": "target/llm-runs/schema/t_013_spacetime_sum_type/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_013_spacetime_sum_type/csharp/server/gpt-5/llm", "scorer_details": { - "sum_type_row_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "args": [ - 1, - 10 - ], - "golden_db": "schema-t-013-spacetime-sum-type-golden", - "golden_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))", - "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", - "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))", - "query": "SELECT Id, Value FROM Result WHERE Id=1", - "reducer": "SetCircle", - "server": "http://127.0.0.1:41045" - } - }, "schema_parity": { "pass": true, "partial": 1.0, @@ -2836,7 +2942,7 @@ "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41045", + "server": "http://127.0.0.1:41165", "tables_diff": null, "tables_equal": true } @@ -2849,21 +2955,38 @@ "expected": 1, "sql": "SELECT COUNT(*) AS n FROM Result WHERE Id=1" } + }, + "sum_type_row_parity": { + "pass": true, + "partial": 1.0, + "notes": { + "args": [ + 1, + 10 + ], + "golden_db": "schema-t-013-spacetime-sum-type-golden", + "golden_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))", + "llm_db": "schema-t-013-spacetime-sum-type-gpt-5-llm", + "llm_out": "Id | Value ----+-------------------------- 1 | (Circle = (Radius = 10))", + "query": "SELECT Id, Value FROM Result WHERE Id=1", + "reducer": "SetCircle", + "server": "http://127.0.0.1:41165" + } } }, "vendor": "openai", - "started_at": "2026-01-23T20:17:49.057528012Z", - "finished_at": "2026-01-23T20:18:46.199910007Z" + "started_at": "2026-01-24T00:03:41.610651465Z", + "finished_at": "2026-01-24T00:05:18.400719234Z" }, "t_014_elementary_columns": { - "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_014_elementary_columns", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Primitive\", Public = true)]\n public partial struct Primitive\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Count;\n public long Total;\n public float Price;\n public double Ratio;\n public bool Active;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Primitive.Insert(new Primitive\n {\n Id = 1,\n Count = 2,\n Total = 3000000000L,\n Price = 1.5f,\n Ratio = 2.25,\n Active = true,\n Name = \"Alice\"\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Primitive\")]\n public partial struct Primitive\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int Count;\n public long Total;\n public float Price;\n public double Ratio;\n public bool Active;\n public string Name;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Primitive.Insert(new Primitive\n {\n Id = 1,\n Count = 2,\n Total = 3000000000L,\n Price = 1.5f,\n Ratio = 2.25,\n Active = true,\n Name = \"Alice\"\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-014-elementary-columns-golden", @@ -2871,42 +2994,42 @@ "work_dir_golden": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_014_elementary_columns/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { - "pass": true, - "partial": 1.0, + "elementary_columns_row_parity": { + "pass": false, + "partial": 0.0, "notes": { - "golden_db": "schema-t-014-elementary-columns-golden", - "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41045", - "tables_diff": null, - "tables_equal": true + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41165/v1/database/c200abb30b25499293ed5bc7ba50399490b1822a8b306291a5d7e0d29840ec60/sql)\n", + "phase": "sql_golden" } }, "elementary_columns_row_count": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41045/v1/database/c200c583183a5879f79de7a06c7395893137fa8a834f830859f2ee7a4c9e10d4/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41165/v1/database/c2005e15aa9be491e8444b574ff6c04413d8ed2bb5427a2c042dc8cc81fcd58b/sql)\n", "phase": "sql" } }, - "elementary_columns_row_parity": { - "pass": false, - "partial": 0.0, + "schema_parity": { + "pass": true, + "partial": 1.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `primitive`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41045/v1/database/c2008dfd963859de34e7ba657c3416cbcee034a702128f54b0cb0b0b03aa914f/sql)\n", - "phase": "sql_golden" + "golden_db": "schema-t-014-elementary-columns-golden", + "llm_db": "schema-t-014-elementary-columns-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41165", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:18:46.199983893Z", - "finished_at": "2026-01-23T20:19:29.507804243Z" + "started_at": "2026-01-24T00:03:26.186891573Z", + "finished_at": "2026-01-24T00:04:04.039047837Z" }, "t_015_product_type_columns": { - "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_015_product_type_columns", "lang": "csharp", "golden_published": true, @@ -2941,7 +3064,7 @@ "llm_out": "Id | Home | Work | Pos ----+----------------------------------+-----------------------------------+---------------- 1 | (Street = \"1 Main\", Zip = 11111) | (Street = \"2 Broad\", Zip = 22222) | (X = 7, Y = 9)", "query": "SELECT Id, Home, Work, Pos FROM Profile WHERE Id=1", "reducer": "Seed", - "server": "http://127.0.0.1:41045" + "server": "http://127.0.0.1:41165" } }, "schema_parity": { @@ -2952,25 +3075,25 @@ "llm_db": "schema-t-015-product-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41045", + "server": "http://127.0.0.1:41165", "tables_diff": null, "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:18:11.292039093Z", - "finished_at": "2026-01-23T20:18:54.161647622Z" + "started_at": "2026-01-24T00:03:55.909160959Z", + "finished_at": "2026-01-24T00:05:10.850801287Z" }, "t_016_sum_type_columns": { - "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_016_sum_type_columns", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 3, "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Drawing\", Public = true)]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Drawing.Insert(new Drawing\n {\n Id = 1,\n A = new Shape.Circle(new Circle { Radius = 10 }),\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Type]\n public partial struct Circle\n {\n public int Radius;\n }\n\n [SpacetimeDB.Type]\n public partial struct Rectangle\n {\n public int Width;\n public int Height;\n }\n\n [SpacetimeDB.Type]\n public partial record Shape : TaggedEnum<(Circle Circle, Rectangle Rectangle)> { }\n\n [SpacetimeDB.Table(Name = \"Drawing\", Public = true)]\n public partial struct Drawing\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public Shape A;\n public Shape B;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n foreach (var existing in ctx.Db.Drawing.Iter())\n {\n ctx.Db.Drawing.Id.Delete(existing.Id);\n }\n\n ctx.Db.Drawing.Insert(new Drawing\n {\n Id = 1,\n A = new Shape.Circle(new Circle { Radius = 10 }),\n B = new Shape.Rectangle(new Rectangle { Width = 4, Height = 6 })\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-016-sum-type-columns-golden", @@ -2982,7 +3105,7 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41045/v1/database/c200b1fbaa5b44721798298318bab63a7fcd80d50f6e47060d881bfa066b8642/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41165/v1/database/c200758d3d7e090a756802600f5abbdfa92e9f80f78bbe9f16875cc52e63db9f/sql)\n", "phase": "sql" } }, @@ -2994,7 +3117,7 @@ "llm_db": "schema-t-016-sum-type-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41045", + "server": "http://127.0.0.1:41165", "tables_diff": null, "tables_equal": true } @@ -3003,24 +3126,24 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41045/v1/database/c2003b0da13c17c6220fd590cdb5d23015a0e698b01b6c6ecbfd10b5aabbbf4c/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `drawings`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41165/v1/database/c200e49d308cedd0b63e897b1e9231e41c79da83179130463533c2754beabd3c/sql)\n", "phase": "sql_golden" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:16:40.802337426Z", - "finished_at": "2026-01-23T20:18:08.899029179Z" + "started_at": "2026-01-24T00:04:02.368901035Z", + "finished_at": "2026-01-24T00:05:38.729154807Z" }, "t_017_scheduled_columns": { - "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_017_scheduled_columns", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 2, "passed_tests": 1, - "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [PrimaryKey, AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [Reducer]\n public static void Tick(ReducerContext ctx, TickTimer timer)\n {\n // Scheduled tick handler (no-op)\n }\n\n [Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n if (ctx.Db.TickTimer.Count == 0)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n }\n}", + "llm_output": "using System;\nusing SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"TickTimer\", Scheduled = nameof(Tick), ScheduledAt = nameof(ScheduledAt))]\n public partial struct TickTimer\n {\n [SpacetimeDB.PrimaryKey, SpacetimeDB.AutoInc]\n public ulong ScheduledId;\n public ScheduleAt ScheduledAt;\n }\n\n [SpacetimeDB.Reducer]\n public static void Tick(ReducerContext ctx, TickTimer _row)\n {\n }\n\n [SpacetimeDB.Reducer(ReducerKind.Init)]\n public static void Init(ReducerContext ctx)\n {\n ctx.Db.TickTimer.Insert(new TickTimer\n {\n ScheduledId = 0,\n ScheduledAt = new ScheduleAt.Interval(TimeSpan.FromMilliseconds(50))\n });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-017-scheduled-columns-golden", @@ -3036,7 +3159,7 @@ "llm_db": "schema-t-017-scheduled-columns-gpt-5-llm", "reducers_diff": null, "reducers_equal": true, - "server": "http://127.0.0.1:41045", + "server": "http://127.0.0.1:41165", "tables_diff": null, "tables_equal": true } @@ -3045,17 +3168,17 @@ "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41045/v1/database/c2000932373ef07be1f77db87f24b98a9819f6033c40da69c914e654c807ead2/sql)\n", + "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `tick_timer`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41165/v1/database/c20013446ea0691be1033226a60d1a74e75b1615111df0d10aa679ebc52edc7c/sql)\n", "phase": "sql" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:16:40.797164170Z", - "finished_at": "2026-01-23T20:18:08.735264338Z" + "started_at": "2026-01-24T00:03:05.250970806Z", + "finished_at": "2026-01-24T00:04:02.368868621Z" }, "t_018_constraints": { - "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_018_constraints", "lang": "csharp", "golden_published": true, @@ -3070,31 +3193,31 @@ "work_dir_golden": "target/llm-runs/schema/t_018_constraints/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_018_constraints/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "constraints_row_parity_after_seed": { "pass": true, "partial": 1.0, "notes": { + "args": [], "golden_db": "schema-t-018-constraints-golden", + "golden_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", "llm_db": "schema-t-018-constraints-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41045", - "tables_diff": null, - "tables_equal": true + "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", + "query": "SELECT Id, Email, Name FROM Account WHERE Id=1", + "reducer": "Seed", + "server": "http://127.0.0.1:41165" } }, - "constraints_row_parity_after_seed": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "args": [], "golden_db": "schema-t-018-constraints-golden", - "golden_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", "llm_db": "schema-t-018-constraints-gpt-5-llm", - "llm_out": "Id | Email | Name ----+-----------------+--------- 1 | \"a@example.com\" | \"Alice\"", - "query": "SELECT Id, Email, Name FROM Account WHERE Id=1", - "reducer": "Seed", - "server": "http://127.0.0.1:41045" + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41165", + "tables_diff": null, + "tables_equal": true } }, "constraints_seed_two_rows": { @@ -3108,11 +3231,11 @@ } }, "vendor": "openai", - "started_at": "2026-01-23T20:16:40.794071431Z", - "finished_at": "2026-01-23T20:18:11.291994106Z" + "started_at": "2026-01-24T00:02:24.983294531Z", + "finished_at": "2026-01-24T00:03:05.250921900Z" }, "t_019_many_to_many": { - "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_019_many_to_many", "lang": "csharp", "golden_published": true, @@ -3127,135 +3250,98 @@ "work_dir_golden": "target/llm-runs/schema/t_019_many_to_many/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_019_many_to_many/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "m2m_has_1_20": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "schema-t-019-many-to-many-golden", - "llm_db": "schema-t-019-many-to-many-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41045", - "tables_diff": null, - "tables_equal": true + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=20" } }, - "m2m_has_2_20": { + "m2m_has_1_10": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=2 AND GroupId=20" + "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=10" } }, - "memberships_three_rows": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Membership" + "golden_db": "schema-t-019-many-to-many-golden", + "llm_db": "schema-t-019-many-to-many-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41165", + "tables_diff": null, + "tables_equal": true } }, - "m2m_has_1_20": { + "memberships_three_rows": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=20" + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM Membership" } }, - "m2m_has_1_10": { + "m2m_has_2_20": { "pass": true, "partial": 1.0, "notes": { "actual": 1, "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=1 AND GroupId=10" + "sql": "SELECT COUNT(*) AS n FROM Membership WHERE UserId=2 AND GroupId=20" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:18:08.899074821Z", - "finished_at": "2026-01-23T20:19:10.793593915Z" + "started_at": "2026-01-24T00:02:35.245089256Z", + "finished_at": "2026-01-24T00:03:26.186843126Z" }, "t_020_ecs": { - "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_020_ecs", "lang": "csharp", - "golden_published": true, + "golden_published": false, "model_name": "GPT-5", - "total_tests": 5, - "passed_tests": 1, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Entity\", Public = true)]\n public partial struct Entity\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n }\n\n [SpacetimeDB.Table(Name = \"Position\", Public = true)]\n public partial struct Position\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Table(Name = \"Velocity\", Public = true)]\n public partial struct Velocity\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int VX;\n public int VY;\n }\n\n [SpacetimeDB.Table(Name = \"NextPosition\", Public = true)]\n public partial struct NextPosition\n {\n [SpacetimeDB.PrimaryKey]\n public int EntityId;\n public int X;\n public int Y;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n // Entity 1\n if (ctx.Db.Entity.Id.Find(1) is not Entity)\n {\n ctx.Db.Entity.Insert(new Entity { Id = 1 });\n }\n if (ctx.Db.Position.EntityId.Find(1) is Position p1)\n {\n p1.X = 0;\n p1.Y = 0;\n ctx.Db.Position.EntityId.Update(p1);\n }\n else\n {\n ctx.Db.Position.Insert(new Position { EntityId = 1, X = 0, Y = 0 });\n }\n if (ctx.Db.Velocity.EntityId.Find(1) is Velocity v1)\n {\n v1.VX = 1;\n v1.VY = 0;\n ctx.Db.Velocity.EntityId.Update(v1);\n }\n else\n {\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 1, VX = 1, VY = 0 });\n }\n\n // Entity 2\n if (ctx.Db.Entity.Id.Find(2) is not Entity)\n {\n ctx.Db.Entity.Insert(new Entity { Id = 2 });\n }\n if (ctx.Db.Position.EntityId.Find(2) is Position p2)\n {\n p2.X = 10;\n p2.Y = 0;\n ctx.Db.Position.EntityId.Update(p2);\n }\n else\n {\n ctx.Db.Position.Insert(new Position { EntityId = 2, X = 10, Y = 0 });\n }\n if (ctx.Db.Velocity.EntityId.Find(2) is Velocity v2)\n {\n v2.VX = -2;\n v2.VY = 3;\n ctx.Db.Velocity.EntityId.Update(v2);\n }\n else\n {\n ctx.Db.Velocity.Insert(new Velocity { EntityId = 2, VX = -2, VY = 3 });\n }\n }\n\n [SpacetimeDB.Reducer]\n public static void Step(ReducerContext ctx)\n {\n foreach (var pos in ctx.Db.Position.Iter())\n {\n var vOpt = ctx.Db.Velocity.EntityId.Find(pos.EntityId);\n if (vOpt is Velocity vel)\n {\n int nx = pos.X + vel.VX;\n int ny = pos.Y + vel.VY;\n\n var npOpt = ctx.Db.NextPosition.EntityId.Find(pos.EntityId);\n if (npOpt is NextPosition np)\n {\n np.X = nx;\n np.Y = ny;\n ctx.Db.NextPosition.EntityId.Update(np);\n }\n else\n {\n ctx.Db.NextPosition.Insert(new NextPosition\n {\n EntityId = pos.EntityId,\n X = nx,\n Y = ny\n });\n }\n }\n }\n }\n}", + "total_tests": 1, + "passed_tests": 0, + "llm_output": null, "category": "schema", "route_api_model": "gpt-5", - "golden_db": "schema-t-020-ecs-golden", - "llm_db": "schema-t-020-ecs-gpt-5-llm", - "work_dir_golden": "target/llm-runs/schema/t_020_ecs/csharp/server/golden", - "work_dir_llm": "target/llm-runs/schema/t_020_ecs/csharp/server/gpt-5/llm", + "golden_db": null, + "llm_db": null, + "work_dir_golden": null, + "work_dir_llm": null, "scorer_details": { - "ecs_next_pos_entity1": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41045/v1/database/c2005b7d5d9b2b6b33a4a581ebd8661a3b0380c077f59eee8e552ef3c67ff1b9/sql)\n", - "phase": "sql" - } - }, - "ecs_seed_positions_count": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41045/v1/database/c2005b7d5d9b2b6b33a4a581ebd8661a3b0380c077f59eee8e552ef3c67ff1b9/sql)\n", - "phase": "sql" - } - }, - "ecs_next_pos_entity2": { - "pass": false, - "partial": 0.0, - "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41045/v1/database/c2005b7d5d9b2b6b33a4a581ebd8661a3b0380c077f59eee8e552ef3c67ff1b9/sql)\n", - "phase": "sql" - } - }, - "ecs_step_next_positions_count": { + "publish_error": { "pass": false, "partial": 0.0, "notes": { - "error": "spacetime sql failed:\nWARNING: This command is UNSTABLE and subject to breaking changes.\n\nError: no such table: `next_positions`. If the table exists, it may be marked private.\n\nCaused by:\n HTTP status client error (400 Bad Request) for url (http://127.0.0.1:41045/v1/database/c2005b7d5d9b2b6b33a4a581ebd8661a3b0380c077f59eee8e552ef3c67ff1b9/sql)\n", - "phase": "sql" - } - }, - "schema_parity": { - "pass": true, - "partial": 1.0, - "notes": { - "golden_db": "schema-t-020-ecs-golden", - "llm_db": "schema-t-020-ecs-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41045", - "tables_diff": null, - "tables_equal": true + "error": "LLM call timed out", + "phase": "build_or_publish" } } }, "vendor": "openai", - "started_at": "2026-01-23T20:18:08.902649901Z", - "finished_at": "2026-01-23T20:19:23.573927579Z" + "started_at": "2026-01-24T00:05:38.729220337Z", + "finished_at": "2026-01-24T00:05:38.729220337Z" }, "t_021_multi_column_index": { - "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "task": "t_021_multi_column_index", "lang": "csharp", "golden_published": true, "model_name": "GPT-5", "total_tests": 4, "passed_tests": 4, - "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Log\")]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = [\"UserId\", \"Day\"])]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Log.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.Log.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.Log.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}", + "llm_output": "using SpacetimeDB;\n\npublic static partial class Module\n{\n [SpacetimeDB.Table(Name = \"Log\", Public = true)]\n [SpacetimeDB.Index.BTree(Name = \"by_user_day\", Columns = new[] { \"UserId\", \"Day\" })]\n public partial struct Log\n {\n [SpacetimeDB.PrimaryKey]\n public int Id;\n public int UserId;\n public int Day;\n public string Message;\n }\n\n [SpacetimeDB.Reducer]\n public static void Seed(ReducerContext ctx)\n {\n ctx.Db.Log.Insert(new Log { Id = 1, UserId = 7, Day = 1, Message = \"a\" });\n ctx.Db.Log.Insert(new Log { Id = 2, UserId = 7, Day = 2, Message = \"b\" });\n ctx.Db.Log.Insert(new Log { Id = 3, UserId = 9, Day = 1, Message = \"c\" });\n }\n}", "category": "schema", "route_api_model": "gpt-5", "golden_db": "schema-t-021-multi-column-index-golden", @@ -3263,26 +3349,22 @@ "work_dir_golden": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/golden", "work_dir_llm": "target/llm-runs/schema/t_021_multi_column_index/csharp/server/gpt-5/llm", "scorer_details": { - "schema_parity": { + "mcindex_lookup_u7_d2": { "pass": true, "partial": 1.0, "notes": { - "golden_db": "schema-t-021-multi-column-index-golden", - "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", - "reducers_diff": null, - "reducers_equal": true, - "server": "http://127.0.0.1:41045", - "tables_diff": null, - "tables_equal": true + "actual": 1, + "expected": 1, + "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=2" } }, - "mcindex_lookup_u7_d2": { + "mcindex_seed_count": { "pass": true, "partial": 1.0, "notes": { - "actual": 1, - "expected": 1, - "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=2" + "actual": 3, + "expected": 3, + "sql": "SELECT COUNT(*) AS n FROM Log" } }, "mcindex_lookup_u7_d1": { @@ -3294,19 +3376,23 @@ "sql": "SELECT COUNT(*) AS n FROM Log WHERE UserId=7 AND Day=1" } }, - "mcindex_seed_count": { + "schema_parity": { "pass": true, "partial": 1.0, "notes": { - "actual": 3, - "expected": 3, - "sql": "SELECT COUNT(*) AS n FROM Log" + "golden_db": "schema-t-021-multi-column-index-golden", + "llm_db": "schema-t-021-multi-column-index-gpt-5-llm", + "reducers_diff": null, + "reducers_equal": true, + "server": "http://127.0.0.1:41165", + "tables_diff": null, + "tables_equal": true } } }, "vendor": "openai", - "started_at": "2026-01-23T20:16:40.799865241Z", - "finished_at": "2026-01-23T20:17:49.057471707Z" + "started_at": "2026-01-24T00:02:23.314785957Z", + "finished_at": "2026-01-24T00:03:16.504408864Z" } } } diff --git a/docs/llms/docs-benchmark-summary.json b/docs/llms/docs-benchmark-summary.json index 8e7f8400b57..aca87b1ed44 100644 --- a/docs/llms/docs-benchmark-summary.json +++ b/docs/llms/docs-benchmark-summary.json @@ -1,38 +1,38 @@ { "version": 1, - "generated_at": "2026-01-23T20:21:59.670Z", + "generated_at": "2026-01-24T00:05:38.779Z", "by_language": { "csharp": { "modes": { "docs": { - "hash": "0bdd5572150f1bebf8032e2007d3a7644df903777de6c1ad070cab80e77b3ed2", + "hash": "b308f98bb4ae39f4777789293eac6c8de59c7354bcca261bda4d8a14888e0a6f", "models": { "GPT-5": { "categories": { "basics": { "tasks": 12, "total_tests": 27, - "passed_tests": 27, - "pass_pct": 100.0, - "task_pass_equiv": 12.0, - "task_pass_pct": 100.0 + "passed_tests": 26, + "pass_pct": 96.296295, + "task_pass_equiv": 11.0, + "task_pass_pct": 91.66667 }, "schema": { "tasks": 10, - "total_tests": 34, - "passed_tests": 25, - "pass_pct": 73.52941, - "task_pass_equiv": 7.3666663, - "task_pass_pct": 73.666664 + "total_tests": 30, + "passed_tests": 24, + "pass_pct": 80.0, + "task_pass_equiv": 7.1666665, + "task_pass_pct": 71.666664 } }, "totals": { "tasks": 22, - "total_tests": 61, - "passed_tests": 52, - "pass_pct": 85.2459, - "task_pass_equiv": 19.366667, - "task_pass_pct": 88.030304 + "total_tests": 57, + "passed_tests": 50, + "pass_pct": 87.7193, + "task_pass_equiv": 18.166666, + "task_pass_pct": 82.57576 } } } @@ -42,34 +42,34 @@ "rust": { "modes": { "docs": { - "hash": "315390340d7551db2791f061b9bff7602dc13350b7a4caee6aca61ef97867ec9", + "hash": "7bd6defbdd71a539817dc6743ff7a4cdcb340cfca98eff7c26a27f44a2a58c5f", "models": { "GPT-5": { "categories": { "basics": { "tasks": 12, - "total_tests": 27, - "passed_tests": 5, - "pass_pct": 18.518518, - "task_pass_equiv": 1.3333334, - "task_pass_pct": 11.111112 + "total_tests": 24, + "passed_tests": 4, + "pass_pct": 16.666666, + "task_pass_equiv": 1.0833334, + "task_pass_pct": 9.027779 }, "schema": { "tasks": 10, - "total_tests": 14, - "passed_tests": 0, - "pass_pct": 0.0, - "task_pass_equiv": 0.0, - "task_pass_pct": 0.0 + "total_tests": 34, + "passed_tests": 8, + "pass_pct": 23.529411, + "task_pass_equiv": 2.05, + "task_pass_pct": 20.5 } }, "totals": { "tasks": 22, - "total_tests": 41, - "passed_tests": 5, - "pass_pct": 12.195122, - "task_pass_equiv": 1.3333334, - "task_pass_pct": 6.060606 + "total_tests": 58, + "passed_tests": 12, + "pass_pct": 20.689655, + "task_pass_equiv": 3.1333334, + "task_pass_pct": 14.242424 } } } @@ -82,10 +82,10 @@ "basics": { "tasks": 12, "total_tests": 27, - "passed_tests": 25, - "pass_pct": 92.59259, - "task_pass_equiv": 10.0, - "task_pass_pct": 83.33333 + "passed_tests": 22, + "pass_pct": 81.48148, + "task_pass_equiv": 8.916667, + "task_pass_pct": 74.30556 }, "schema": { "tasks": 10, @@ -99,10 +99,10 @@ "totals": { "tasks": 22, "total_tests": 61, - "passed_tests": 51, - "pass_pct": 83.60656, - "task_pass_equiv": 17.533333, - "task_pass_pct": 79.69697 + "passed_tests": 48, + "pass_pct": 78.68852, + "task_pass_equiv": 16.45, + "task_pass_pct": 74.772736 } } } diff --git a/sdks/csharp/.gitignore b/sdks/csharp/.gitignore index 42502a20a58..249ecdec52d 100644 --- a/sdks/csharp/.gitignore +++ b/sdks/csharp/.gitignore @@ -76,3 +76,8 @@ obj~ /nuget.config /nuget.config.meta .idea/ + +# Hydrated SDK DLLs (produced by `cargo ci dlls`) +/packages/ +/packages.meta +!/packages/.gitignore diff --git a/sdks/csharp/packages/spacetimedb.bsatn.runtime.meta b/sdks/csharp/packages/spacetimedb.bsatn.runtime.meta deleted file mode 100644 index 6dc8b62571c..00000000000 --- a/sdks/csharp/packages/spacetimedb.bsatn.runtime.meta +++ /dev/null @@ -1,8 +0,0 @@ -fileFormatVersion: 2 -guid: 8865417631feca343997ae22b3320e75 -folderAsset: yes -DefaultImporter: - externalObjects: {} - userData: - assetBundleName: - assetBundleVariant: diff --git a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0.meta b/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0.meta deleted file mode 100644 index a0714721e1a..00000000000 --- a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0.meta +++ /dev/null @@ -1,8 +0,0 @@ -fileFormatVersion: 2 -guid: ae5666f23a6d73c43b030a1b9ba5916b -folderAsset: yes -DefaultImporter: - externalObjects: {} - userData: - assetBundleName: - assetBundleVariant: diff --git a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers.meta b/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers.meta deleted file mode 100644 index aff342ea1f5..00000000000 --- a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers.meta +++ /dev/null @@ -1,8 +0,0 @@ -fileFormatVersion: 2 -guid: f112b25d371a72548b6b4959e104c4c3 -folderAsset: yes -DefaultImporter: - externalObjects: {} - userData: - assetBundleName: - assetBundleVariant: diff --git a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers/dotnet.meta b/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers/dotnet.meta deleted file mode 100644 index f2031f11a56..00000000000 --- a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers/dotnet.meta +++ /dev/null @@ -1,8 +0,0 @@ -fileFormatVersion: 2 -guid: 8554a9e409be36a42bdb316f4677fabf -folderAsset: yes -DefaultImporter: - externalObjects: {} - userData: - assetBundleName: - assetBundleVariant: diff --git a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers/dotnet/cs.meta b/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers/dotnet/cs.meta deleted file mode 100644 index a387186fa76..00000000000 --- a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers/dotnet/cs.meta +++ /dev/null @@ -1,8 +0,0 @@ -fileFormatVersion: 2 -guid: a24e03405bfe85e4996b2524189ac24a -folderAsset: yes -DefaultImporter: - externalObjects: {} - userData: - assetBundleName: - assetBundleVariant: diff --git a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers/dotnet/cs/SpacetimeDB.BSATN.Codegen.dll b/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers/dotnet/cs/SpacetimeDB.BSATN.Codegen.dll deleted file mode 100755 index 1db1c1f2115..00000000000 Binary files a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers/dotnet/cs/SpacetimeDB.BSATN.Codegen.dll and /dev/null differ diff --git a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers/dotnet/cs/SpacetimeDB.BSATN.Codegen.dll.meta b/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers/dotnet/cs/SpacetimeDB.BSATN.Codegen.dll.meta deleted file mode 100644 index fcd3478e3fb..00000000000 --- a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/analyzers/dotnet/cs/SpacetimeDB.BSATN.Codegen.dll.meta +++ /dev/null @@ -1,71 +0,0 @@ -fileFormatVersion: 2 -guid: 31ff25d2b5723480e873b2b8f46bba86 -labels: -- RoslynAnalyzer -PluginImporter: - externalObjects: {} - serializedVersion: 2 - iconMap: {} - executionOrder: {} - defineConstraints: [] - isPreloaded: 0 - isOverridable: 1 - isExplicitlyReferenced: 0 - validateReferences: 1 - platformData: - - first: - : Any - second: - enabled: 0 - settings: - Exclude Editor: 1 - Exclude Linux64: 1 - Exclude OSXUniversal: 1 - Exclude Win: 1 - Exclude Win64: 1 - - first: - Any: - second: - enabled: 0 - settings: {} - - first: - Editor: Editor - second: - enabled: 0 - settings: - CPU: AnyCPU - DefaultValueInitialized: true - OS: AnyOS - - first: - Standalone: Linux64 - second: - enabled: 0 - settings: - CPU: None - - first: - Standalone: OSXUniversal - second: - enabled: 0 - settings: - CPU: None - - first: - Standalone: Win - second: - enabled: 0 - settings: - CPU: None - - first: - Standalone: Win64 - second: - enabled: 0 - settings: - CPU: None - - first: - Windows Store Apps: WindowsStoreApps - second: - enabled: 0 - settings: - CPU: AnyCPU - userData: - assetBundleName: - assetBundleVariant: diff --git a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/lib.meta b/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/lib.meta deleted file mode 100644 index 3050ce8ab83..00000000000 --- a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/lib.meta +++ /dev/null @@ -1,8 +0,0 @@ -fileFormatVersion: 2 -guid: dc6d364d8264f4d43847a2f962630f96 -folderAsset: yes -DefaultImporter: - externalObjects: {} - userData: - assetBundleName: - assetBundleVariant: diff --git a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/lib/netstandard2.1.meta b/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/lib/netstandard2.1.meta deleted file mode 100644 index 56e591b6926..00000000000 --- a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/lib/netstandard2.1.meta +++ /dev/null @@ -1,8 +0,0 @@ -fileFormatVersion: 2 -guid: b964bd4b6540b3c4e90d73d46afd8fc2 -folderAsset: yes -DefaultImporter: - externalObjects: {} - userData: - assetBundleName: - assetBundleVariant: diff --git a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/lib/netstandard2.1/SpacetimeDB.BSATN.Runtime.dll b/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/lib/netstandard2.1/SpacetimeDB.BSATN.Runtime.dll deleted file mode 100755 index b66c06e7b8a..00000000000 Binary files a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/lib/netstandard2.1/SpacetimeDB.BSATN.Runtime.dll and /dev/null differ diff --git a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/lib/netstandard2.1/SpacetimeDB.BSATN.Runtime.dll.meta b/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/lib/netstandard2.1/SpacetimeDB.BSATN.Runtime.dll.meta deleted file mode 100644 index de2769892f6..00000000000 --- a/sdks/csharp/packages/spacetimedb.bsatn.runtime/1.5.0/lib/netstandard2.1/SpacetimeDB.BSATN.Runtime.dll.meta +++ /dev/null @@ -1,33 +0,0 @@ -fileFormatVersion: 2 -guid: 8ab8f6f35b91340e6b1269b6448e3bf0 -PluginImporter: - externalObjects: {} - serializedVersion: 2 - iconMap: {} - executionOrder: {} - defineConstraints: [] - isPreloaded: 0 - isOverridable: 1 - isExplicitlyReferenced: 0 - validateReferences: 1 - platformData: - - first: - Any: - second: - enabled: 1 - settings: {} - - first: - Editor: Editor - second: - enabled: 0 - settings: - DefaultValueInitialized: true - - first: - Windows Store Apps: WindowsStoreApps - second: - enabled: 0 - settings: - CPU: AnyCPU - userData: - assetBundleName: - assetBundleVariant: