Vectordata streaming support for examples/bench [Updated]

.github/workflows/run-bench.yml

@@ -140,7 +140,7 @@ jobs:
         env:
           DATASET_HASH: ${{ secrets.DATASETS_KEYPATH }}
         run: |
-          # Check if jvector-examples directory and AutoBenchYAML class exist
+          # Check if jvector-examples directory exists
           if [ ! -d "jvector-examples" ]; then
             echo "Warning: jvector-examples directory not found in branch ${{ matrix.branch }}. Skipping benchmark."
             exit 0
@@ -201,12 +201,12 @@ jobs:
             java ${{ matrix.jdk >= 20 && '--enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector' || '' }} \
               ${{ matrix.jdk >= 22 && '-Djvector.experimental.enable_native_vectorization=true' || '' }} \
               -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp/heap_dump/ -Xmx${HALF_MEM_GB}g \
-              -cp jvector-examples/target/jvector-examples-*-jar-with-dependencies.jar io.github.jbellis.jvector.example.AutoBenchYAML --output ${SAFE_BRANCH}-bench-results ${CONFIG_ARG} dpr-1M
+              -cp jvector-examples/target/jvector-examples-*-jar-with-dependencies.jar io.github.jbellis.jvector.benchframe.BenchFrameCLI autobenchyaml --output ${SAFE_BRANCH}-bench-results dpr-1M
           else
             java ${{ matrix.jdk >= 20 && '--enable-native-access=ALL-UNNAMED --add-modules=jdk.incubator.vector' || '' }} \
               ${{ matrix.jdk >= 22 && '-Djvector.experimental.enable_native_vectorization=true' || '' }} \
               -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/tmp/heap_dump/ -Xmx${HALF_MEM_GB}g \
-              -cp jvector-examples/target/jvector-examples-*-jar-with-dependencies.jar io.github.jbellis.jvector.example.AutoBenchYAML --output ${SAFE_BRANCH}-bench-results ${CONFIG_ARG}${BENCH_SUFFIX:+ }${BENCH_ARG}
+              -cp jvector-examples/target/jvector-examples-*-jar-with-dependencies.jar io.github.jbellis.jvector.benchframe.BenchFrameCLI autobenchyaml --output ${SAFE_BRANCH}-bench-results${BENCH_SUFFIX:+ }${BENCH_ARG}
           fi
 
           # Move the results to the benchmark_results directory

.github/workflows/unit-tests.yaml

@@ -59,21 +59,21 @@ jobs:
       - name: Test Panama Support (JDK ${{ matrix.jdk }})
         if: matrix.jdk == '20'
         run: >-
-          mvn -B -Pjdk20 -pl jvector-tests test -am test
+          mvn -X -B -Pjdk20 -pl jvector-tests test -am test
           -DTest_RequireSpecificVectorizationProvider=PanamaVectorizationProvider
 
       - name: Verify native-access vector support (JDK ${{ matrix.jdk }})
         if: matrix.jdk == '24'
         run: >-
-          mvn -B -Punix-amd64-profile -pl jvector-tests -am test
+          mvn -X -B -Punix-amd64-profile -pl jvector-tests -am test
           -DTest_RequireSpecificVectorizationProvider=NativeVectorizationProvider
           -Dsurefire.failIfNoSpecifiedTests=false
           -Dtest=TestVectorizationProvider
 
       - name: Compile, run tests and package (JDK ${{ matrix.jdk }})
         if: matrix.jdk == '24'
         run: >-
-          mvn -B -Punix-amd64-profile -pl jvector-tests -am test
+          mvn -X -B -Punix-amd64-profile -pl jvector-tests -am test
           -DTest_RequireSpecificVectorizationProvider=NativeVectorizationProvider
 
       - name: Test Summary for (ISA:${{ matrix.isa}},JDK${{ matrix.jdk }})

.gitignore

@@ -34,6 +34,16 @@ hdf5/
 ### aider
 .aider*
 
+### claude
+.claude/**
+
+### junie
+.junie/**
+
 # JMH generated files
 dependency-reduced-pom.xml
 results.csv
+
+# Local testing files
+local/**
+

benchmarks-jmh/pom.xml

@@ -94,6 +94,21 @@
                     </execution>
                 </executions>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-javadoc-plugin</artifactId>
+                <configuration>
+                    <additionalJOptions>
+                        <additionalJOption>--add-modules=jdk.incubator.vector</additionalJOption>
+                    </additionalJOptions>
+                    <release>22</release>
+                    <detectOfflineLinks>false</detectOfflineLinks>
+                    <includeDependencySources>true</includeDependencySources>
+                    <dependencySourceIncludes>
+                        <dependencySourceInclude>io.github.jbellis:*</dependencySourceInclude>
+                    </dependencySourceIncludes>
+                </configuration>
+            </plugin>
         </plugins>
     </build>
 </project>

benchmarks-jmh/src/main/java/io/github/jbellis/jvector/bench/PQDistanceCalculationBenchmark.java

@@ -23,15 +23,13 @@
 import io.github.jbellis.jvector.quantization.PQVectors;
 import io.github.jbellis.jvector.quantization.ProductQuantization;
 import io.github.jbellis.jvector.vector.VectorSimilarityFunction;
-import io.github.jbellis.jvector.vector.VectorUtil;
 import io.github.jbellis.jvector.vector.VectorizationProvider;
-import io.github.jbellis.jvector.vector.types.ByteSequence;
 import io.github.jbellis.jvector.vector.types.VectorFloat;
 import io.github.jbellis.jvector.vector.types.VectorTypeSupport;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.openjdk.jmh.annotations.*;
 import org.openjdk.jmh.infra.Blackhole;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -49,7 +47,7 @@
 @Measurement(iterations = 3)
 @Threads(1)
 public class PQDistanceCalculationBenchmark {
-    private static final Logger log = LoggerFactory.getLogger(PQDistanceCalculationBenchmark.class);
+    private static final Logger log = LogManager.getLogger(PQDistanceCalculationBenchmark.class);
     private static final VectorTypeSupport VECTOR_TYPE_SUPPORT = VectorizationProvider.getInstance().getVectorTypeSupport();
     private final VectorSimilarityFunction vsf = VectorSimilarityFunction.EUCLIDEAN;
 

jvector-examples/pom.xml

@@ -13,7 +13,9 @@
     <name>JVector Examples</name>
     <properties>
         <awssdk.version>2.21.10</awssdk.version>
+        <datatools.version>0.1.10</datatools.version>
     </properties>
+
     <build>
         <plugins>
             <plugin>
@@ -27,27 +29,45 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-assembly-plugin</artifactId>
-                <version>3.6.0</version>
-                <configuration>
-                    <descriptorRefs>
-                        <descriptorRef>jar-with-dependencies</descriptorRef>
-                    </descriptorRefs>
-                    <archive>
-                        <manifest>
-                            <mainClass>io.github.jbellis.jvector.example.AutoBenchYAML</mainClass>
-                        </manifest>
-                    </archive>
-                </configuration>
+                <version>3.7.1</version>
                 <executions>
                     <execution>
-                        <id>make-assembly</id>
                         <phase>package</phase>
                         <goals>
                             <goal>single</goal>
                         </goals>
                     </execution>
                 </executions>
+                <configuration>
+                    <archive>
+                        <manifest>
+                            <addClasspath>true</addClasspath>
+                            <mainClass>io.github.jbellis.jvector.benchframe.BenchFrameCLI</mainClass>
+                        </manifest>
+                    </archive>
+                    <descriptorRefs>
+                        <!--                  <ref>assembly.xml</ref>-->
+                        <descriptorRef>jar-with-dependencies</descriptorRef>
+                        <!--            <descriptorRef>assembly</descriptorRef>-->
+                    </descriptorRefs>
+                </configuration>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-javadoc-plugin</artifactId>
+                <configuration>
+                    <additionalJOptions>
+                        <additionalJOption>--add-modules=jdk.incubator.vector</additionalJOption>
+                    </additionalJOptions>
+                    <release>22</release>
+                    <detectOfflineLinks>false</detectOfflineLinks>
+                    <includeDependencySources>true</includeDependencySources>
+                    <dependencySourceIncludes>
+                        <dependencySourceInclude>io.github.jbellis:*</dependencySourceInclude>
+                    </dependencySourceIncludes>
+                </configuration>
+            </plugin>
+
         </plugins>
     </build>
     <dependencies>
@@ -72,17 +92,6 @@
                 </exclusion>
             </exclusions>
         </dependency>
-        <!-- Add SLF4J and Logback for logging -->
-        <dependency>
-            <groupId>org.slf4j</groupId>
-            <artifactId>slf4j-api</artifactId>
-            <version>2.0.9</version>
-        </dependency>
-        <dependency>
-            <groupId>ch.qos.logback</groupId>
-            <artifactId>logback-classic</artifactId>
-            <version>1.4.11</version>
-        </dependency>
         <dependency>
             <groupId>software.amazon.awssdk</groupId>
             <artifactId>s3-transfer-manager</artifactId>
@@ -115,38 +124,40 @@
             </exclusions>
         </dependency>
         <dependency>
-            <groupId>com.kohlschutter.junixsocket</groupId>
-            <artifactId>junixsocket-core</artifactId>
-            <version>2.8.1</version>
-            <type>pom</type>
+            <groupId>io.nosqlbench</groupId>
+            <artifactId>datatools-vectordata</artifactId>
+            <version>${datatools.version}</version>
         </dependency>
         <dependency>
-            <groupId>com.fasterxml.jackson.core</groupId>
-            <artifactId>jackson-databind</artifactId>
-            <version>2.17.1</version>
+            <groupId>io.nosqlbench</groupId>
+            <artifactId>datatools-nbvectors</artifactId>
+            <version>${datatools.version}</version>
         </dependency>
         <dependency>
-            <groupId>junit</groupId>
-            <artifactId>junit</artifactId>
-            <version>4.13.1</version>
-            <scope>test</scope>
+            <groupId>info.picocli</groupId>
+            <artifactId>picocli</artifactId>
+            <version>4.7.6</version>
         </dependency>
+
         <dependency>
-            <groupId>org.testng</groupId>
-            <artifactId>testng</artifactId>
-            <version>7.3.0</version>
-            <scope>test</scope>
+            <groupId>com.kohlschutter.junixsocket</groupId>
+            <artifactId>junixsocket-core</artifactId>
+            <version>2.8.1</version>
+            <type>pom</type>
         </dependency>
         <dependency>
             <groupId>org.junit.jupiter</groupId>
             <artifactId>junit-jupiter-api</artifactId>
-            <version>5.11.4</version>
             <scope>test</scope>
         </dependency>
     </dependencies>
     <profiles>
         <profile>
             <id>jdk11</id>
+            <properties>
+                <maven.compiler.source>11</maven.compiler.source>
+                <maven.compiler.target>11</maven.compiler.target>
+            </properties>
             <build>
                 <plugins>
                     <plugin>
@@ -273,6 +284,9 @@
             <activation>
                 <activeByDefault>true</activeByDefault>
             </activation>
+            <properties>
+                <maven.compiler.target>22</maven.compiler.target>
+            </properties>
             <dependencies>
                 <dependency>
                     <groupId>io.github.jbellis</groupId>