Add type library utility plugin and CLI

Cargo.toml

@@ -25,6 +25,8 @@ members = [
     "plugins/warp/examples/headless",
     "plugins/workflow_objc",
     "plugins/workflow_objc/demo",
+    "plugins/bntl_utils",
+    "plugins/bntl_utils/cli",
 ]
 
 [workspace.dependencies]

about.toml

@@ -9,4 +9,5 @@ accepted = [
     "LicenseRef-scancode-google-patent-license-fuchsia",
     "MPL-2.0",
     "LicenseRef-scancode-unknown-license-reference",
+    "BSD-2-Clause"
 ]

arch/msp430/Cargo.toml

@@ -4,6 +4,7 @@ version = "0.1.0"
 authors = ["jrozner"]
 edition = "2021"
 license = "Apache-2.0"
+publish = false
 
 [dependencies]
 binaryninja.workspace = true

arch/riscv/Cargo.toml

@@ -4,6 +4,7 @@ version = "0.1.0"
 authors = ["Ryan Snyder <ryan.snyder.or@gmail.com>"]
 edition = "2021"
 license = "Apache-2.0"
+publish = false
 
 [dependencies]
 binaryninja.workspace = true

plugins/bntl_utils/CMakeLists.txt

@@ -0,0 +1,168 @@
+cmake_minimum_required(VERSION 3.15 FATAL_ERROR)
+
+project(bntl_utils)
+
+if(NOT BN_API_BUILD_EXAMPLES AND NOT BN_INTERNAL_BUILD)
+    if(NOT BN_API_PATH)
+        # If we have not already defined the API source directory try and find it.
+        find_path(
+                BN_API_PATH
+                NAMES binaryninjaapi.h
+                # List of paths to search for the clone of the api
+                HINTS ../../.. ../../binaryninja/api/ binaryninjaapi binaryninja-api $ENV{BN_API_PATH}
+                REQUIRED
+        )
+    endif()
+    set(CARGO_STABLE_VERSION 1.91.1)
+    add_subdirectory(${BN_API_PATH} binaryninjaapi)
+endif()
+
+file(GLOB_RECURSE PLUGIN_SOURCES CONFIGURE_DEPENDS
+        ${PROJECT_SOURCE_DIR}/Cargo.toml
+        ${PROJECT_SOURCE_DIR}/src/*.rs)
+
+if(CMAKE_BUILD_TYPE MATCHES Debug)
+    if(DEMO)
+        set(TARGET_DIR ${PROJECT_BINARY_DIR}/target/dev-demo)
+        set(CARGO_OPTS --target-dir=${PROJECT_BINARY_DIR}/target --profile=dev-demo)
+    else()
+        set(TARGET_DIR ${PROJECT_BINARY_DIR}/target/debug)
+        set(CARGO_OPTS --target-dir=${PROJECT_BINARY_DIR}/target)
+    endif()
+else()
+    if(DEMO)
+        set(TARGET_DIR ${PROJECT_BINARY_DIR}/target/release-demo)
+        set(CARGO_OPTS --target-dir=${PROJECT_BINARY_DIR}/target --profile=release-demo)
+    else()
+        set(TARGET_DIR ${PROJECT_BINARY_DIR}/target/release)
+        set(CARGO_OPTS --target-dir=${PROJECT_BINARY_DIR}/target --release)
+    endif()
+endif()
+
+if(FORCE_COLORED_OUTPUT)
+    set(CARGO_OPTS ${CARGO_OPTS} --color always)
+endif()
+
+if(DEMO)
+    set(CARGO_FEATURES --features demo --manifest-path ${PROJECT_SOURCE_DIR}/demo/Cargo.toml)
+
+    set(OUTPUT_FILE_NAME ${CMAKE_STATIC_LIBRARY_PREFIX}${PROJECT_NAME}_static${CMAKE_STATIC_LIBRARY_SUFFIX})
+    set(OUTPUT_PDB_NAME ${CMAKE_STATIC_LIBRARY_PREFIX}${PROJECT_NAME}.pdb)
+    set(OUTPUT_FILE_PATH ${CMAKE_BINARY_DIR}/${OUTPUT_FILE_NAME})
+    set(OUTPUT_PDB_PATH ${CMAKE_BINARY_DIR}/${OUTPUT_PDB_NAME})
+
+    set(BINJA_LIB_DIR $<TARGET_FILE_DIR:binaryninjacore>)
+else()
+    # NOTE: --no-default-features is set to disable building artifacts used for testing
+    # NOTE: the linker is looking in the target dir and linking on it apparently.
+    set(CARGO_FEATURES "--no-default-features")
+
+    set(OUTPUT_FILE_NAME ${CMAKE_SHARED_LIBRARY_PREFIX}${PROJECT_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX})
+    set(OUTPUT_PDB_NAME ${CMAKE_SHARED_LIBRARY_PREFIX}${PROJECT_NAME}.pdb)
+    set(OUTPUT_FILE_PATH ${BN_CORE_PLUGIN_DIR}/${OUTPUT_FILE_NAME})
+    set(OUTPUT_PDB_PATH ${BN_CORE_PLUGIN_DIR}/${OUTPUT_PDB_NAME})
+
+    set(BINJA_LIB_DIR ${BN_INSTALL_BIN_DIR})
+endif()
+
+
+add_custom_target(${PROJECT_NAME} ALL DEPENDS ${OUTPUT_FILE_PATH})
+add_dependencies(${PROJECT_NAME} binaryninjaapi)
+get_target_property(BN_API_SOURCE_DIR binaryninjaapi SOURCE_DIR)
+list(APPEND CMAKE_MODULE_PATH "${BN_API_SOURCE_DIR}/cmake")
+find_package(BinaryNinjaCore REQUIRED)
+
+set_property(TARGET ${PROJECT_NAME} PROPERTY OUTPUT_FILE_PATH ${OUTPUT_FILE_PATH})
+
+# Add the whole api to the depends too
+file(GLOB API_SOURCES CONFIGURE_DEPENDS
+        ${BN_API_SOURCE_DIR}/binaryninjacore.h
+        ${BN_API_SOURCE_DIR}/rust/src/*.rs
+        ${BN_API_SOURCE_DIR}/rust/binaryninjacore-sys/src/*.rs)
+
+find_program(RUSTUP_PATH rustup REQUIRED HINTS ~/.cargo/bin)
+set(RUSTUP_COMMAND ${RUSTUP_PATH} run ${CARGO_STABLE_VERSION} cargo)
+
+if(APPLE)
+    if(UNIVERSAL)
+        if(CMAKE_BUILD_TYPE MATCHES Debug)
+            if(DEMO)
+                set(AARCH64_LIB_PATH ${PROJECT_BINARY_DIR}/target/aarch64-apple-darwin/dev-demo/${OUTPUT_FILE_NAME})
+                set(X86_64_LIB_PATH ${PROJECT_BINARY_DIR}/target/x86_64-apple-darwin/dev-demo/${OUTPUT_FILE_NAME})
+            else()
+                set(AARCH64_LIB_PATH ${PROJECT_BINARY_DIR}/target/aarch64-apple-darwin/debug/${OUTPUT_FILE_NAME})
+                set(X86_64_LIB_PATH ${PROJECT_BINARY_DIR}/target/x86_64-apple-darwin/debug/${OUTPUT_FILE_NAME})
+            endif()
+        else()
+            if(DEMO)
+                set(AARCH64_LIB_PATH ${PROJECT_BINARY_DIR}/target/aarch64-apple-darwin/release-demo/${OUTPUT_FILE_NAME})
+                set(X86_64_LIB_PATH ${PROJECT_BINARY_DIR}/target/x86_64-apple-darwin/release-demo/${OUTPUT_FILE_NAME})
+            else()
+                set(AARCH64_LIB_PATH ${PROJECT_BINARY_DIR}/target/aarch64-apple-darwin/release/${OUTPUT_FILE_NAME})
+                set(X86_64_LIB_PATH ${PROJECT_BINARY_DIR}/target/x86_64-apple-darwin/release/${OUTPUT_FILE_NAME})
+            endif()
+        endif()
+
+        add_custom_command(
+                OUTPUT ${OUTPUT_FILE_PATH}
+                COMMAND ${CMAKE_COMMAND} -E env
+                MACOSX_DEPLOYMENT_TARGET=10.14 BINARYNINJADIR=${BINJA_LIB_DIR}
+                ${RUSTUP_COMMAND} clean --target=aarch64-apple-darwin ${CARGO_OPTS} --package binaryninjacore-sys
+                COMMAND ${CMAKE_COMMAND} -E env
+                MACOSX_DEPLOYMENT_TARGET=10.14 BINARYNINJADIR=${BINJA_LIB_DIR}
+                ${RUSTUP_COMMAND} clean --target=x86_64-apple-darwin ${CARGO_OPTS} --package binaryninjacore-sys
+                COMMAND ${CMAKE_COMMAND} -E env
+                MACOSX_DEPLOYMENT_TARGET=10.14 BINARYNINJADIR=${BINJA_LIB_DIR}
+                ${RUSTUP_COMMAND} build --target=aarch64-apple-darwin ${CARGO_OPTS} ${CARGO_FEATURES}
+                COMMAND ${CMAKE_COMMAND} -E env
+                MACOSX_DEPLOYMENT_TARGET=10.14 BINARYNINJADIR=${BINJA_LIB_DIR}
+                ${RUSTUP_COMMAND} build --target=x86_64-apple-darwin ${CARGO_OPTS} ${CARGO_FEATURES}
+                COMMAND lipo -create ${AARCH64_LIB_PATH} ${X86_64_LIB_PATH} -output ${OUTPUT_FILE_PATH}
+                WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+                DEPENDS ${PLUGIN_SOURCES} ${API_SOURCES}
+        )
+    else()
+        add_custom_command(
+                OUTPUT ${OUTPUT_FILE_PATH}
+                COMMAND ${CMAKE_COMMAND} -E env
+                MACOSX_DEPLOYMENT_TARGET=10.14 BINARYNINJADIR=${BINJA_LIB_DIR}
+                ${RUSTUP_COMMAND} clean ${CARGO_OPTS} --package binaryninjacore-sys
+                COMMAND ${CMAKE_COMMAND} -E env
+                MACOSX_DEPLOYMENT_TARGET=10.14 BINARYNINJADIR=${BINJA_LIB_DIR}
+                ${RUSTUP_COMMAND} build ${CARGO_OPTS} ${CARGO_FEATURES}
+                COMMAND ${CMAKE_COMMAND} -E copy ${TARGET_DIR}/${OUTPUT_FILE_NAME} ${OUTPUT_FILE_PATH}
+                WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+                DEPENDS ${PLUGIN_SOURCES} ${API_SOURCES}
+        )
+    endif()
+elseif(WIN32)
+    if(DEMO)
+        add_custom_command(
+                OUTPUT ${OUTPUT_FILE_PATH}
+                COMMAND ${CMAKE_COMMAND} -E env BINARYNINJADIR=${BINJA_LIB_DIR} ${RUSTUP_COMMAND} clean ${CARGO_OPTS} --package binaryninjacore-sys
+                COMMAND ${CMAKE_COMMAND} -E env BINARYNINJADIR=${BINJA_LIB_DIR} ${RUSTUP_COMMAND} build ${CARGO_OPTS} ${CARGO_FEATURES}
+                COMMAND ${CMAKE_COMMAND} -E copy ${TARGET_DIR}/${OUTPUT_FILE_NAME} ${OUTPUT_FILE_PATH}
+                WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+                DEPENDS ${PLUGIN_SOURCES} ${API_SOURCES}
+        )
+    else()
+        add_custom_command(
+                OUTPUT ${OUTPUT_FILE_PATH}
+                COMMAND ${CMAKE_COMMAND} -E env BINARYNINJADIR=${BINJA_LIB_DIR} ${RUSTUP_COMMAND} clean ${CARGO_OPTS} --package binaryninjacore-sys
+                COMMAND ${CMAKE_COMMAND} -E env BINARYNINJADIR=${BINJA_LIB_DIR} ${RUSTUP_COMMAND} build ${CARGO_OPTS} ${CARGO_FEATURES}
+                COMMAND ${CMAKE_COMMAND} -E copy ${TARGET_DIR}/${OUTPUT_FILE_NAME} ${OUTPUT_FILE_PATH}
+                COMMAND ${CMAKE_COMMAND} -E copy ${TARGET_DIR}/${OUTPUT_PDB_NAME} ${OUTPUT_PDB_PATH}
+                WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+                DEPENDS ${PLUGIN_SOURCES} ${API_SOURCES}
+        )
+    endif()
+else()
+    add_custom_command(
+            OUTPUT ${OUTPUT_FILE_PATH}
+            COMMAND ${CMAKE_COMMAND} -E env BINARYNINJADIR=${BINJA_LIB_DIR} ${RUSTUP_COMMAND} clean ${CARGO_OPTS} --package binaryninjacore-sys
+            COMMAND ${CMAKE_COMMAND} -E env BINARYNINJADIR=${BINJA_LIB_DIR} ${RUSTUP_COMMAND} build ${CARGO_OPTS} ${CARGO_FEATURES}
+            COMMAND ${CMAKE_COMMAND} -E copy ${TARGET_DIR}/${OUTPUT_FILE_NAME} ${OUTPUT_FILE_PATH}
+            WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
+            DEPENDS ${PLUGIN_SOURCES} ${API_SOURCES}
+    )
+endif()

plugins/bntl_utils/Cargo.toml

@@ -0,0 +1,40 @@
+[package]
+name = "bntl_utils"
+version = "0.1.0"
+edition = "2021"
+license = "Apache-2.0"
+publish = false
+
+[lib]
+crate-type = ["cdylib", "lib"]
+
+[dependencies]
+binaryninja.workspace = true
+binaryninjacore-sys.workspace = true
+tracing = "0.1"
+thiserror = "2.0"
+similar = "2.7.0"
+serde = { version = "1.0", features = ["derive"] }
+serde_json = "1.0"
+tempdir = "0.3"
+nt-apiset = "0.1.0"
+url = "2.5"
+uuid = "1.20"
+walkdir = "2.5"
+dashmap = "6.1"
+
+# For reports
+minijinja = "2.10.2"
+minijinja-embed = "2.10.2"
+
+[build-dependencies]
+minijinja-embed = "2.10.2"
+
+# TODO: We need to depend on latest because the windows-metadata crate has not yet been bumped, but depending on the crate
+# TODO: with git will mean we pull in all of the data of the crate instead of just the necessary bits, we likely need to
+# TODO: wait until the windows-metadata crate is bumped before merging this PR.
+# TODO: Relevant PR: https://github.com/microsoft/windows-rs/pull/3799
+# TODO: Relevant issue: https://github.com/microsoft/windows-rs/issues/3887
+[dependencies.windows-metadata]
+git = "https://github.com/microsoft/windows-rs"
+tag = "72"

plugins/bntl_utils/README.md

@@ -0,0 +1,5 @@
+# BNTL Utilities
+
+A plugin and CLI tool for processing Binary Ninja type libraries (BNTL).
+
+For CLI build instructions and usage see [here](./cli/README.md).

plugins/bntl_utils/build.rs

@@ -0,0 +1,48 @@
+use std::path::PathBuf;
+
+fn main() {
+    let link_path = std::env::var_os("DEP_BINARYNINJACORE_PATH")
+        .expect("DEP_BINARYNINJACORE_PATH not specified");
+
+    println!("cargo::rustc-link-lib=dylib=binaryninjacore");
+    println!("cargo::rustc-link-search={}", link_path.to_str().unwrap());
+
+    #[cfg(target_os = "linux")]
+    {
+        println!(
+            "cargo::rustc-link-arg=-Wl,-rpath,{0},-L{0}",
+            link_path.to_string_lossy()
+        );
+    }
+
+    #[cfg(target_os = "macos")]
+    {
+        let crate_name = std::env::var("CARGO_PKG_NAME").expect("CARGO_PKG_NAME not set");
+        let lib_name = crate_name.replace('-', "_");
+        println!(
+            "cargo::rustc-link-arg=-Wl,-install_name,@rpath/lib{}.dylib",
+            lib_name
+        );
+    }
+
+    let out_dir = std::env::var("OUT_DIR").expect("OUT_DIR specified");
+    let out_dir_path = PathBuf::from(out_dir);
+
+    // Copy all binaries to OUT_DIR for unit tests.
+    let bin_dir: PathBuf = "fixtures/".into();
+    if let Ok(entries) = std::fs::read_dir(bin_dir) {
+        for entry in entries {
+            let entry = entry.unwrap();
+            let path = entry.path();
+            if path.is_file() {
+                let file_name = path.file_name().unwrap();
+                let dest_path = out_dir_path.join(file_name);
+                std::fs::copy(&path, &dest_path).expect("failed to copy binary to OUT_DIR");
+            }
+        }
+    }
+
+    println!("cargo::rerun-if-changed=src/templates");
+    // Templates used for rendering reports.
+    minijinja_embed::embed_templates!("src/templates");
+}

plugins/bntl_utils/cli/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "bntl_cli"
+version = "0.1.0"
+edition = "2024"
+
+[dependencies]
+binaryninja.workspace = true
+binaryninjacore-sys.workspace = true
+bntl_utils = { path = "../" }
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+clap = { version = "4.5.58", features = ["derive"] }
+rayon = "1.11"
+serde_json = "1.0"
+thiserror = "2.0"

plugins/bntl_utils/cli/README.md

@@ -0,0 +1,61 @@
+# Headless BNTL Processor
+
+Provides headless support for generating, inspecting, and validating Binary Ninja type libraries (BNTL).
+
+### Building
+
+> Assuming you have the following:
+> - A compatible Binary Ninja with headless usage (see [this documentation](https://docs.binary.ninja/dev/batch.html#batch-processing-and-other-automation-tips) for more information)
+> - Clang
+> - Rust (currently tested for 1.91.1)
+> - Set `BINARYNINJADIR` env variable to your installation directory (see [here](https://docs.binary.ninja/guide/#binary-path) for more details)
+    >   - If this is not set, the -sys crate will try and locate using the default installation path and last run location.
+
+1. Clone this repository (`git clone https://github.com/Vector35/binaryninja-api/tree/dev`)
+2. Build in release (`cargo build --release`)
+
+If compilation fails because it could not link against binaryninjacore than you should double-check you set `BINARYNINJADIR` correctly.
+
+Once it finishes you now will have a `bntl_cli` binary in `target/release` for use.
+
+### Usage
+
+> Assuming you already have the `bntl_cli` binary and a valid headless compatible Binary Ninja license.
+
+#### Create
+
+Generate a new type library from local files or remote projects.
+
+Examples:
+
+- `./bntl_cli create sqlite3.dll "windows-x86_64" ./headers/ ./output/`
+    - Places a single `sqlite.dll.bntl` file in the `output` directory, as headers have no dependency names associated they will be named `sqlite.dll`.
+- `./bntl_cli create myproject "windows-x86_64" binaryninja://enterprise/https://enterprise.com/23ce5eaa-f532-4a93-80f2-a7d7f0aed040/ ./output/`
+    - Downloads and processes all files in the project, placing potentially multiple `.bntl` files in the `output` directory.
+- `./bntl_cli create sqlite3.dll "windows-x86_64" ./winmd/ ./output/`
+  - `winmd` files are also supported as input, they will be processed together. You also probably want to provide some apiset schema files as well.
+
+#### Dump
+
+Export a type library back into a C header file for inspection.
+
+Examples:
+
+- `./bntl_cli dump sqlite3.dll.bntl ./output/sqlite.h`
+
+#### Diff
+
+Compare two type libraries and generate a .diff file containing a similarity ratio.
+
+Examples:
+
+- `./bntl_cli diff sqlite3.dll.bntl sqlite3.dll.bntl ./output/sqlite.diff`
+
+#### Validate
+
+Check type libraries for common errors, ensuring all referenced types exist across specified platforms.
+
+Examples:
+
+- `./bntl_cli validate ./typelibs/ ./output/`
+  - Pass in a directory containing `.bntl` files to validate, outputting a JSON file for each type library containing any errors.

plugins/bntl_utils/cli/build.rs

@@ -0,0 +1,15 @@
+fn main() {
+    let link_path = std::env::var_os("DEP_BINARYNINJACORE_PATH")
+        .expect("DEP_BINARYNINJACORE_PATH not specified");
+
+    println!("cargo::rustc-link-lib=dylib=binaryninjacore");
+    println!("cargo::rustc-link-search={}", link_path.to_str().unwrap());
+
+    #[cfg(not(target_os = "windows"))]
+    {
+        println!(
+            "cargo::rustc-link-arg=-Wl,-rpath,{0},-L{0}",
+            link_path.to_string_lossy()
+        );
+    }
+}