From 58f3f91daef39f9226b0f9f1347e373090d7f24a Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Mon, 6 Oct 2025 14:57:53 +0200 Subject: [PATCH 01/13] 4x --- CMakeLists.txt | 58 +++-- environment-dev.yml | 4 +- include/pyjs/post_js/fixes.js | 8 + include/pyjs/pre_js/dynload/dynload.js | 294 +++++++++---------------- include/pyjs/pre_js/init.js | 76 +++++-- include/pyjs/pre_js/load_pkg.js | 66 +++++- src/export_js_module.cpp | 62 ++++-- src/js_timestamp.cpp | 2 +- src/untar.cpp | 7 + 9 files changed, 311 insertions(+), 266 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 6ea2e36..f0ac6b5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -115,14 +115,16 @@ target_link_libraries(pyjs PRIVATE ${PYTHON_UTIL_LIBS} pybind11::embed) target_compile_options(pyjs PUBLIC --std=c++17 + PUBLIC -msimd128 PUBLIC -Wno-deprecated - PUBLIC "SHELL: -fexceptions" + PUBLIC "SHELL: -fwasm-exceptions" ) target_link_options(pyjs - PRIVATE -lembind PUBLIC -Wno-unused-command-line-argument - PUBLIC "SHELL: -fexceptions" + PUBLIC "SHELL: -fwasm-exceptions" + PUBLIC "SHELL: -lembind" + # PUBLIC "SHELL: -fsanitize=address" #PUBLIC "SHELL:-s EXPORT_EXCEPTION_HANDLING_HELPERS" #PUBLIC "SHELL:-s EXCEPTION_CATCHING_ALLOWED=['we only want to allow exception handling in side modules']" ) @@ -198,44 +200,54 @@ target_link_libraries(pyjs_runtime_browser PRIVATE pyjs pybind11::embed ${PYTHO target_compile_options(pyjs_runtime_browser PUBLIC --std=c++17 PUBLIC -Wno-deprecated + PUBLIC -msimd128 PUBLIC "SHELL: -s ENVIRONMENT=${ENVIRONMENT}" - PUBLIC "SHELL: -fexceptions" - #PUBLIC "SHELL:-s EXPORT_EXCEPTION_HANDLING_HELPERS" + PUBLIC "SHELL: -fwasm-exceptions" + PUBLIC "SHELL: -sSUPPORT_LONGJMP" + PUBLIC "SHELL: -s EXPORT_EXCEPTION_HANDLING_HELPERS" PUBLIC "SHELL: -s FORCE_FILESYSTEM" - PUBLIC "SHELL: -s LZ4=1" - PUBLIC "SHELL: -flto" - # PUBLIC "SHELL: -s WASM_BIGINT" + PUBLIC "SHELL: -s EXPORT_ALL=1" + # PUBLIC "SHELL: -flto" + # PUBLIC "SHELL: -s MAXIMUM_MEMORY=2GB" + # PUBLIC "SHELL: -fsanitize=address" ) target_link_options(pyjs_runtime_browser - PRIVATE -lembind PUBLIC -Wno-unused-command-line-argument PUBLIC "SHELL: -s MODULARIZE=1" PUBLIC "SHELL: -s EXPORT_NAME=\"createModule\"" - PUBLIC "SHELL: -s EXPORT_ES6=0" - PUBLIC "SHELL: -s USE_ES6_IMPORT_META=0" - PUBLIC "SHELL: -s DEMANGLE_SUPPORT=0" - PUBLIC "SHELL: -s ASSERTIONS=0" + PUBLIC "SHELL: -s EXPORT_ALL=1" + PUBLIC "SHELL: -s EXPORT_EXCEPTION_HANDLING_HELPERS" + # PUBLIC "SHELL: -s EXPORT_ES6=0" + # PUBLIC "SHELL: -s DEMANGLE_SUPPORT=0" + # PUBLIC "SHELL: -s ASSERTIONS=0" PUBLIC "SHELL: -s ALLOW_MEMORY_GROWTH=1" - PUBLIC "SHELL: -s EXIT_RUNTIME=1" - PUBLIC "SHELL: -s WASM=1" + # PUBLIC "SHELL: -s EXIT_RUNTIME=1" + PUBLIC "SHELL: -s INITIAL_MEMORY=20971520" + PUBLIC "SHELL: -s MAXIMUM_MEMORY=4GB" + # PUBLIC "SHELL: -s WASM=1" PUBLIC "SHELL: -s USE_PTHREADS=0" PUBLIC "SHELL: -s ENVIRONMENT=${ENVIRONMENT}" - PUBLIC "SHELL: -fexceptions" + PUBLIC "SHELL: -fwasm-exceptions" + PUBLIC "SHELL: -sSUPPORT_LONGJMP" PUBLIC "SHELL: -s MAIN_MODULE=1" - PUBLIC "SHELL: -s ENVIRONMENT=${ENVIRONMENT}" - PUBLIC "SHELL: -s TOTAL_STACK=16mb" - PUBLIC "SHELL: -s INITIAL_MEMORY=64mb" + # PUBLIC "SHELL: -s MAXIMUM_MEMORY=2GB" PUBLIC "SHELL: -s FORCE_FILESYSTEM" - PUBLIC "SHELL: -s LZ4=1" PUBLIC "SHELL: --post-js pyjs_post.js" PUBLIC "SHELL: --pre-js pyjs_pre.js" - PUBLIC "SHELL: -flto" - PUBLIC "SHELL: -lidbfs.js" - PUBLIC "SHELL: -s WASM_BIGINT" + # PUBLIC "SHELL: -flto" + # PUBLIC "SHELL: -lidbfs.js" + PUBLIC "SHELL: -lembind" + # PUBLIC "SHELL: -fsanitize=address" ) +# target_link_options(pyjs_runtime_browser PRIVATE +# "-sEXPORTED_FUNCTIONS=['_emscripten_dlopen_promise','_dlerror','FS']" +# ) + + + install(TARGETS pyjs_runtime_browser DESTINATION ${CMAKE_INSTALL_PREFIX}/lib_js/pyjs) install(FILES diff --git a/environment-dev.yml b/environment-dev.yml index 676239d..bcf1601 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -1,7 +1,9 @@ name: pyjs-wasm channels: - conda-forge + - /Users/thorstenbeier/src/recipes/output - https://repo.prefix.dev/emscripten-forge-dev + dependencies: - cmake - pip @@ -27,7 +29,7 @@ dependencies: - notebook >=7,<8 # to include the extension to switch between JupyterLab and Notebook # pyjs_code_runner dev deps - hatchling - - emscripten_emscripten-wasm32 + - emscripten_emscripten-wasm32 == 4.0.9 - pip: - JLDracula - pyjs_code_runner >= 3.0 diff --git a/include/pyjs/post_js/fixes.js b/include/pyjs/post_js/fixes.js index 3260295..32060e6 100644 --- a/include/pyjs/post_js/fixes.js +++ b/include/pyjs/post_js/fixes.js @@ -7,3 +7,11 @@ Module['PATH'] = PATH Module['LDSO'] = LDSO Module['getDylinkMetadata'] = getDylinkMetadata Module['loadDynamicLibrary'] = loadDynamicLibrary +Module['getPromise'] = getPromise +Module['promiseMap'] = promiseMap + +Module['stackSave'] = stackSave +Module['stackRestore'] = stackRestore +Module['runtimeKeepalivePush'] = runtimeKeepalivePush +Module['UTF8ToString'] = UTF8ToString +Module['stringToUTF8OnStack'] = stringToUTF8OnStack \ No newline at end of file diff --git a/include/pyjs/pre_js/dynload/dynload.js b/include/pyjs/pre_js/dynload/dynload.js index 8ee2482..34426f1 100644 --- a/include/pyjs/pre_js/dynload/dynload.js +++ b/include/pyjs/pre_js/dynload/dynload.js @@ -1,17 +1,3 @@ -const memoize = (fn) => { - let cache = {}; - return (...args) => { - let n = args[0]; - if (n in cache) { - return cache[n]; - } else { - let result = fn(n); - cache[n] = result; - return result; - } - }; -}; - function createLock() { let _lock = Promise.resolve(); @@ -26,215 +12,139 @@ function createLock() { return acquireLock; } -function isInSharedLibraryPath(prefix, libPath){ - if (libPath.startsWith("/")){ - const dirname = libPath.substring(0, libPath.lastIndexOf("/")); - if(prefix == "/"){ - return (dirname == `/lib`); - } - else{ - return (dirname == `${prefix}/lib`); - } - } - else{ - return false; + _lock = createLock(); + + + + // * local + // * global + // * undefined // when we dont + function libraryType (path) { + console.log("check lib type for", path); + if (path.includes("cpython-3") && path.includes("-wasm32-emscripten.so")) { + return "local"; } -} + return undefined; + } + + function getFilenameFromPath(path) { + const parts = path.split("/"); + return parts[parts.length - 1]; + } async function loadDynlibsFromPackage( prefix, python_version, pkg_file_name, - pkg_is_shared_library, dynlibPaths, - ) { - - // for(const path of dynlibPaths){ - // console.log(path); - // } - - // assume that shared libraries of a package are located in .libs directory, - // following the convention of auditwheel. - if(prefix == "/"){ - var sitepackages = `/lib/python${python_version[0]}.${python_version[1]}/site-packages` - } - else{ - var sitepackages = `${prefix}/lib/python${python_version[0]}.${python_version[1]}/site-packages` - } - const auditWheelLibDir = `${sitepackages}/${ - pkg_file_name.split("-")[0] - }.libs`; - - // This prevents from reading large libraries multiple times. - const readFileMemoized = memoize(Module.FS.readFile); - - const forceGlobal = !!pkg_is_shared_library; - - - - let dynlibs = []; - - if (forceGlobal) { - dynlibs = dynlibPaths.map((path) => { - return { - path: path, - global: true, - }; - }); - } else { - const globalLibs = calculateGlobalLibs( - dynlibPaths, - readFileMemoized, - ); - - dynlibs = dynlibPaths.map((path) => { - const global = globalLibs.has(Module.PATH.basename(path)); - return { - path: path, - global: global || !! pkg_is_shared_library || isInSharedLibraryPath(prefix, path) || path.startsWith(auditWheelLibDir), - }; - }); - } - - dynlibs.sort((lib1, lib2) => Number(lib2.global) - Number(lib1.global)); - - for (const { path, global } of dynlibs) { - await loadDynlib(prefix, path, global, [auditWheelLibDir], readFileMemoized); - } - } - -function createDynlibFS( - prefix, - lib, - searchDirs, - readFileFunc ) { - const dirname = lib.substring(0, lib.lastIndexOf("/")); - let _searchDirs = searchDirs || []; - if(prefix == "/"){ - _searchDirs = _searchDirs.concat([dirname], [`/lib`]); - } - else{ - _searchDirs = _searchDirs.concat([dirname], [`${prefix}/lib`]); - } - - const resolvePath = (path) => { - //console.log("resolvePath", path); - - if (Module.PATH.basename(path) !== Module.PATH.basename(lib)) { - //console.debug(`Searching a library from ${path}, required by ${lib}`); + // if(pkg_file_name == "zlib") { + // console.log("zlib is always loaded, skipping loading from package"); + // return; + // } + if(prefix != "/") + { + throw `only root prefix / is supported for loading shared libraries from packages, got: ${prefix}`; + } + + console.log("LAPALUZA"); + + // console.log("load shared objects from package:", pkg_file_name, dynlibPaths); + for (let i = 0; i < dynlibPaths.length; i++) { + let path = dynlibPaths[i]; + + // // if the name contains "_tests" we skip loading the shared libraries + // if (path.includes("_tests") || path.includes("_simd")) { + // console.log(`skipping loading shared libraries from package ${pkg_file_name} because it is a test package or simd package`); + // continue; + // } + + const isSymlink = FS.isLink(FS.lstat(path)); + if (isSymlink) { + console.log(`skipping symlinked shared library ${path} from package ${pkg_file_name}`); + continue; } - for (const dir of _searchDirs) { - const fullPath = Module.PATH.join2(dir, path); - //console.log("SERARCHING", fullPath); - if (Module.FS.findObject(fullPath) !== null) { - //console.log("FOUND", fullPath); - return fullPath; - } - } - return path; - }; + const releaseDynlibLock = await _lock(); + try { - let readFile = (path) => - Module.FS.readFile(resolvePath(path)); + //#define RTLD_LAZY 1 + // #define RTLD_NOW 2 + // #define RTLD_NOLOAD 4 + // #define RTLD_NODELETE 4096 + // #define RTLD_GLOBAL 256 + // #define RTLD_LOCAL 0 - if (readFileFunc !== undefined) { - readFile = (path) => readFileFunc(resolvePath(path)); - } + const libt = libraryType(path); + let flag = 2; // RTLD_NOW + // if(libt === "local"){ + // console.log(`loading local shared library ${path} from package ${pkg_file_name}`); + // flag = 2 /* RTLD_NOW */ | 0 /* RTLD_LOCAL */; + // } + // else if(libt === "global"){ + // console.log(`loading global shared library ${path} from package ${pkg_file_name}`); + // flag = 2 /* RTLD_NOW */ | 256 /* RTLD_GLOBAL */; + // } + // else{ + // console.log(`loading shared library ${path} from package ${pkg_file_name} with default flags`); + // } - const fs = { - findObject: (path, dontResolveLastLink) => { - let obj = Module.FS.findObject(resolvePath(path), dontResolveLastLink); + const stack = Module.stackSave(); + const pathUTF8 = Module.stringToUTF8OnStack(path); + try { + const pid = Module._emscripten_dlopen_promise(pathUTF8, flag); + Module.stackRestore(stack); + const promise = Module.getPromise(pid); + Module.promiseMap.free(pid); + // time it + const start = performance.now(); - if (obj === null) { - console.debug(`Failed to find a library: ${resolvePath(path)}`); - } + + await promise; - return obj; - }, - readFile: readFile, - }; - return fs; -} -function calculateGlobalLibs( - libs, - readFileFunc -) { - let readFile = Module.FS.readFile; - if (readFileFunc !== undefined) { - readFile = readFileFunc; - } + const end = performance.now(); - const globalLibs = new Set(); - libs.forEach((lib) => { - const binary = readFile(lib); - const needed = Module.getDylinkMetadata(binary).neededDynlibs; - needed.forEach((lib) => { - globalLibs.add(lib); - }); - }); - return globalLibs; -} + } catch (e) { + const dll_error_ptr = Module._dlerror(); + if (dll_error_ptr === 0) { + throw Error("unknown error loading shared library"); + } + const error = Module.UTF8ToString( + dll_error_ptr, + 512, // Use enough space for the error message + ); + const error_msg = error.trim(); -// Emscripten has a lock in the corresponding code in library_browser.js. I -// don't know why we need it, but quite possibly bad stuff will happen without -// it. -const acquireDynlibLock = createLock(); + throw new Error(`error loading shared library ${path} from package ${pkg_file_name}: ${error_msg}`); + + } -async function loadDynlib(prefix, lib, global, searchDirs, readFileFunc) { - if (searchDirs === undefined) { - searchDirs = []; - } - const releaseDynlibLock = await acquireDynlibLock(); - - try { - const fs = createDynlibFS(prefix, lib, searchDirs, readFileFunc); - - const libName = Module.PATH.basename(lib); - - // contains cpython-3 and with wasm32-emscripten - const is_cython_lib = libName.includes("cpython-3") && libName.includes("wasm32-emscripten"); - - // load cython library from full path - const load_name = is_cython_lib ? lib : libName; - - await Module.loadDynamicLibrary(load_name, { - loadAsync: true, - nodelete: true, - allowUndefined: true, - global: global && !is_cython_lib, - fs: fs - }) - - const dsoOnlyLibName = Module.LDSO.loadedLibsByName[libName]; - const dsoFullLib = Module.LDSO.loadedLibsByName[lib]; - - if(!dsoOnlyLibName && !dsoFullLib){ - console.execption(`Failed to load ${libName} from ${lib} LDSO not found`); - } + if(libt !== "local"){ + console.log('re-register shared library with filename without path'); + // get filename from path + const filename = getFilenameFromPath(path); + console.log(`registering shared library ${path} from package ${pkg_file_name} as ${filename} in global namespace`); + // Module.LDSO.loadedLibsByName[filename] = Module.LDSO.loadedLibsByName[path]; - if(!is_cython_lib){ - if (!dsoOnlyLibName) { - Module.LDSO.loadedLibsByName[libName] = dsoFullLib - } - - if(!dsoFullLib){ - Module.LDSO.loadedLibsByName[lib] = dsoOnlyLibName; } + console.log(`!!!!!!!!!shared library ${path} from package ${pkg_file_name} is ready`); + } catch (e) { + throw e; + } finally { + console.log("release dynlib lock"); + releaseDynlibLock(); } - } finally { - releaseDynlibLock(); + } } diff --git a/include/pyjs/pre_js/init.js b/include/pyjs/pre_js/init.js index 9b11133..5eabd4d 100644 --- a/include/pyjs/pre_js/init.js +++ b/include/pyjs/pre_js/init.js @@ -19,32 +19,59 @@ Module['init_phase_1'] = async function(prefix, python_version, verbose) { var p = await Module['_wait_run_dependencies'](); if(prefix == "/"){ - Module.setenv("PYTHONHOME", `/`); - Module.setenv("PYTHONPATH", `/lib/python${version_str}/site-packages:/usr/lib/python${version_str}`); - + Module.setenv("LANG", "en_US.UTF-8"); + + // LC_COLLATE="C" + // LC_CTYPE="UTF-8" + // LC_MESSAGES="C" + // LC_MONETARY="C" + // LC_NUMERIC="C" + // LC_TIME="C" + // Module.setenv("LC_COLLATE", "C"); + // Module.setenv("LC_CTYPE", "UTF-8"); + // Module.setenv("LC_MESSAGES", "C"); + // Module.setenv("LC_MONETARY", "C"); + // Module.setenv("LC_NUMERIC", "C"); + // Module.setenv("LC_TIME", "C"); + + const pypath = `/lib/python${version_str}/site-packages:/usr/lib/python${version_str}/site-packages`; var side_path = `/lib/python${version_str}/site-packages`; + + console.log("PYTHONPATH",pypath); + console.log("SIDE_PATH",side_path); + Module.setenv("PYTHONHOME", `/`); + Module.setenv("PYTHONPATH", pypath); + Module.create_directories(side_path); } else{ - Module.setenv("PYTHONHOME", prefix); - Module.setenv("PYTHONPATH", `${prefix}/lib/python${version_str}/site-packages:/usr/lib/python${version_str}`); - var side_path = `${prefix}/lib/python${version_str}/site-packages`; + throw new Error("prefix must be / in wasm build"); } - Module.create_directories(side_path); - Module["_interpreter"] = new Module["_Interpreter"]() - var default_scope = Module["main_scope"]() - Module["default_scope"] = default_scope; + // Module["_interpreter"] = new Module["_Interpreter"]() + console.log("initialize interpreter"); + Module["_initialize_interpreter"](); + console.log("interpreter initialized"); + + + + default_scope = Module["globals"]() + // Module["default_scope"] = default_scope; + // Module['_py_objects'].push(Module["default_scope"]); + - Module['_py_objects'].push(Module["default_scope"]); Module['_py_objects'].push(Module["_interpreter"]); - Module['exec'] = function(code, globals=default_scope, locals=default_scope) { - let ret = Module._exec(code, globals, locals) + Module['exec'] = function(code, globals=default_scope) { + if(globals === undefined){ + globals = Module["globals"]() + } + let ret = Module._exec(code, globals) + // console.error("exec done"); if (ret.has_err) { throw ret } @@ -52,8 +79,11 @@ Module['init_phase_1'] = async function(prefix, python_version, verbose) { - Module['eval'] = function(code, globals=default_scope, locals=default_scope) { - let ret = Module._eval(code, globals, locals) + Module['eval'] = function(code, globals=default_scope) { + if(globals === undefined){ + globals = Module["globals"]() + } + let ret = Module._eval(code, globals) if (ret.has_err) { throw ret } else { @@ -61,8 +91,11 @@ Module['init_phase_1'] = async function(prefix, python_version, verbose) { } }; - Module['eval_file'] = function(file, globals=default_scope, locals=default_scope) { - let ret = Module._eval_file(file, globals, locals) + Module['eval_file'] = function(file, globals=default_scope) { + if(globals === undefined){ + globals = Module["globals"]() + } + let ret = Module._eval_file(file, globals) if (ret.has_err) { throw ret } @@ -127,14 +160,14 @@ Module['init_phase_1'] = async function(prefix, python_version, verbose) { let types = keys.map(Module['_get_type_string']) - let ret = this._raw_getitem(keys, types, keys.length) + let ret = this._raw_getitem(keys, types, BigInt(keys.length)) if (ret.has_err) { throw ret } else { return ret['ret'] } }; - if(verbose){console.log("in init phase 2 done");} + if(verbose){console.log("in init phase 1 done!!");} } Module['init_phase_2'] = function(prefix, python_version, verbose) { @@ -174,7 +207,7 @@ except Exception as e: return await Module._py_async_exec_eval.py_call(script, globals, locals) } if(verbose){console.log("assign pyobjects IV");} - Module._add_resolve_done_callback = Module.exec_eval(` + Module.exec(` import asyncio def _add_resolve_done_callback(future, resolve, reject): ensured_future = asyncio.ensure_future(future) @@ -185,8 +218,9 @@ def _add_resolve_done_callback(future, resolve, reject): reject(repr(err)) ensured_future.add_done_callback(done) -_add_resolve_done_callback `) + + Module._add_resolve_done_callback = Module.eval(`_add_resolve_done_callback`) Module._py_objects.push(Module._add_resolve_done_callback); diff --git a/include/pyjs/pre_js/load_pkg.js b/include/pyjs/pre_js/load_pkg.js index 9b89651..093685e 100644 --- a/include/pyjs/pre_js/load_pkg.js +++ b/include/pyjs/pre_js/load_pkg.js @@ -19,8 +19,10 @@ Module["mkdirs"] = function (dirname) { Module["_untar_from_python"] = function(tarball_path, target_dir = "") { + Module.exec("print('lalalala')"); Module.exec(` def _py_untar(tarball_path, target_dir): + print("1") import tarfile import json from pathlib import Path @@ -28,13 +30,14 @@ def _py_untar(tarball_path, target_dir): import shutil import os import sys + print("2") def check_wasm_magic_number(file_path: Path) -> bool: WASM_BINARY_MAGIC = b"\\0asm" with file_path.open(mode="rb") as file: return file.read(4) == WASM_BINARY_MAGIC - + print("3") target_dir = target_dir if target_dir == "": @@ -63,8 +66,9 @@ def _py_untar(tarball_path, target_dir): raise e return s `) + console.log("calling into _py_untar") let shared_libs = Module.eval(`_py_untar("${tarball_path}", "${target_dir}")`) - + return JSON.parse(shared_libs) } @@ -247,8 +251,11 @@ Module["bootstrap_from_empack_packed_environment"] = async function let version = python_package.version.split(".").map(x => parseInt(x)); - if(verbose){console.log("start init_phase_1");} + await Module.init_phase_1(prefix, version, verbose); + + console.log("bootstrapping python done") + } @@ -276,20 +283,46 @@ Module["bootstrap_from_empack_packed_environment"] = async function let python_version = python_package.version.split(".").map(x => parseInt(x)); // fetch init python itself - console.log("--bootstrap_python"); if(verbose){ console.log("bootstrap_python"); } - let python_is_ready_promise = bootstrap_python(prefix, package_tarballs_root_url, python_package, verbose); + + let python_is_ready_promise = undefined; + try{ + python_is_ready_promise = bootstrap_python(prefix, package_tarballs_root_url, python_package, verbose); + console.log("waiting for python to be ready"); + await python_is_ready_promise; + console.log("python is ready"); + } + catch(e){ + console.log("error while bootstrapping python", e) + throw e + } // create array with size if(verbose){ console.log("fetchAndUntarAll"); } - let shared_libs = await Promise.all([ - ...packages.map(pkg => fetchAndUntar(package_tarballs_root_url, python_is_ready_promise, pkg, verbose)), - ...all_mount_points.map(pkg => fetchAndUntar(package_tarballs_root_url, python_is_ready_promise, pkg, verbose)) - ]); + // let shared_libs = await Promise.all([ + // ...packages.map(pkg => fetchAndUntar(package_tarballs_root_url, python_is_ready_promise, pkg, verbose)), + // ...all_mount_points.map(pkg => fetchAndUntar(package_tarballs_root_url, python_is_ready_promise, pkg, verbose)) + // ]); + + // console.log("shared_libs:", shared_libs); + + + // same as above, but sequentially to debug better + let shared_libs = [] + for(let pkg of packages){ + console.log("fetching and untarring package", pkg.name) + let sl = await fetchAndUntar(package_tarballs_root_url, python_is_ready_promise, pkg, verbose) + shared_libs.push(sl) + } + for(let pkg of all_mount_points){ + console.log("fetching and untarring mount point", pkg.name) + let sl = await fetchAndUntar(package_tarballs_root_url, python_is_ready_promise, pkg, verbose) + shared_libs.push(sl) + } if(verbose){ console.log("init_phase_2"); @@ -299,6 +332,14 @@ Module["bootstrap_from_empack_packed_environment"] = async function if(verbose){ console.log("init shared"); } + + + // in a worker we dont need to pre-load shared libraries + const is_worker = (typeof WorkerGlobalScope !== 'undefined') && (self instanceof WorkerGlobalScope); + if(is_worker){ + // skip_loading_shared_libs = true; + } + if(!skip_loading_shared_libs){ // instantiate all packages for (let i = 0; i < packages.length; i++) { @@ -309,16 +350,19 @@ Module["bootstrap_from_empack_packed_environment"] = async function for (let j = 0; j < shared_libs[i].length; j++) { let sl = shared_libs[i][j]; } + console.log(`loading #${shared_libs[i].length} shared libraries for package ${packages[i].name}`); await Module._loadDynlibsFromPackage( prefix, python_version, packages[i].name, - false, shared_libs[i] - ) + ); + console.log(`asd loading shared libraries for package ${packages[i].name} done`); } } } + console.log("loading shared libraries done PUSH"); + Module.runtimeKeepalivePush(); if(verbose){ console.log("done bootstrapping");} } diff --git a/src/export_js_module.cpp b/src/export_js_module.cpp index 5b15a39..f5522c2 100644 --- a/src/export_js_module.cpp +++ b/src/export_js_module.cpp @@ -12,6 +12,9 @@ #include #include +#include +#include + namespace pyjs { @@ -19,12 +22,13 @@ namespace pyjs namespace py = pybind11; - em::val eval(const std::string & code, py::object & globals, py::object & locals) - { em::val ret = em::val::object(); + em::val eval(const std::string & code, py::object & globals) + { + em::val ret = em::val::object();; + ret.set("has_err", false); try { - py::object py_ret = py::eval(code, globals, locals); - ret.set("has_err",em::val(false)); + py::object py_ret = py::eval(code, globals); auto [jsval, is_proxy] = implicit_py_to_js(py_ret); ret.set("ret",jsval); ret.set("is_proxy",is_proxy); @@ -41,13 +45,13 @@ namespace pyjs - em::val exec(const std::string & code, py::object & globals, py::object & locals) + em::val exec(const std::string & code, py::object & globals) { em::val ret = em::val::object(); + ret.set("has_err", false); try { - py::exec(code, globals, locals); - ret.set("has_err",em::val(false)); + py::exec(code, globals); return ret; } catch (py::error_already_set& e) @@ -55,17 +59,18 @@ namespace pyjs ret.set("has_err",em::val(true)); ret.set("message",em::val(std::string(e.what()))); ret.set("error",em::val(std::string(e.what()))); + return ret; } } - em::val eval_file(const std::string & filename, py::object & globals, py::object & locals) + em::val eval_file(const std::string & filename, py::object & globals) { em::val ret = em::val::object(); try { - py::eval_file(filename, globals, locals); + py::eval_file(filename, globals); ret.set("has_err",em::val(false)); return ret; } @@ -111,6 +116,12 @@ namespace pyjs .constructor<>() ; + em::function("_initialize_interpreter", +[]() + { + py::initialize_interpreter(false, 0, nullptr, false); + } + ); + em::function("_eval", &eval); em::function("_exec", &exec); em::function("_eval_file", &eval_file); @@ -126,14 +137,21 @@ namespace pyjs // py-object (proxy) export_py_object(); - // main scope - em::function("main_scope",em::select_overload( - []()->py::object{ - auto scope = py::module_::import("__main__").attr("__dict__"); - //py::exec("import pyjs;import asyncio", scope); - return scope; - } - )); + // // main scope + // em::function("main_scope",em::select_overload( + // []()->py::object{ + // std::cout<<"get scope"<py::object{ + return py::globals(); + }); em::function("cout", em::select_overload([](const std::string& val) @@ -142,6 +160,16 @@ namespace pyjs em::function("extract_exception_message", &extract_exception_message); + // em::function("_emscripten_dlopen_promise", +[](const std::string& filename, int flags) + // { + // return reinterpret_cast(emscripten_dlopen_promise(filename.c_str(), flags)); + // }); + + // em::function("_dlerror", +[]() + // { + // return std::string(dlerror()); + // }); + } } diff --git a/src/js_timestamp.cpp b/src/js_timestamp.cpp index a1588d5..b6759b5 100644 --- a/src/js_timestamp.cpp +++ b/src/js_timestamp.cpp @@ -1 +1 @@ -#define PYJS_JS_UTC_TIMESTAMP "2025-09-01 06:22:09.224255" \ No newline at end of file +#define PYJS_JS_UTC_TIMESTAMP "2025-09-23 12:30:08.785762" \ No newline at end of file diff --git a/src/untar.cpp b/src/untar.cpp index fbff8af..7127ffc 100644 --- a/src/untar.cpp +++ b/src/untar.cpp @@ -202,6 +202,7 @@ namespace pyjs throw std::runtime_error("untar error: short read error: expected 512"); } if (is_end_of_archive(buff)) { + std::cout<<"End of archive after "< (1 << 30)) { // >1GB is suspicious + std::cout<<"Invalid filesize in tar header: "< 0) { bytes_read = fread(buff, 1, 512, a); if (bytes_read < 512) { @@ -277,6 +283,7 @@ namespace pyjs f = NULL; } } + std::cout<<"HERE WE ARE"< Date: Mon, 6 Oct 2025 15:50:43 +0200 Subject: [PATCH 02/13] 4x --- .github/workflows/docs.yml | 2 +- .github/workflows/main.yml | 8 ++++---- build_mkdocs.sh | 2 +- environment-dev.yml | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 1c679a9..49bf1c3 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -38,7 +38,7 @@ jobs: environment-name: pyjs-wasm condarc: | channels: - - https://repo.mamba.pm/emscripten-forge + - https://repo.mamba.pm/emscripten-forge-4x - conda-forge - name: build the docs diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d3575b5..51bc8aa 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,7 +34,7 @@ jobs: environment-name: pyjs-wasm condarc: | channels: - - https://repo.prefix.dev/emscripten-forge-dev + - https://repo.prefix.dev/ - conda-forge @@ -48,7 +48,7 @@ jobs: micromamba create -n pyjs-build-wasm \ --platform=emscripten-wasm32 \ - -c https://repo.prefix.dev/emscripten-forge-dev\ + -c https://repo.prefix.dev/emscripten-forge-4x\ -c https://repo.mamba.pm/conda-forge \ --yes \ python=${{matrix.python_version}} \ @@ -87,7 +87,7 @@ jobs: micromamba activate pyjs-wasm micromamba create -n pyjs-build-wasm-with-numpy \ --platform=emscripten-wasm32 \ - -c https://repo.prefix.dev/emscripten-forge-dev\ + -c https://repo.prefix.dev/emscripten-forge-4x\ -c https://repo.mamba.pm/conda-forge \ --yes \ "python=${{matrix.python_version}}" pytest numpy exceptiongroup @@ -128,7 +128,7 @@ jobs: micromamba activate pyjs-wasm micromamba create -n pyjs-build-wasm-no-numpy \ --platform=emscripten-wasm32 \ - -c https://repo.prefix.dev/emscripten-forge-dev\ + -c https://repo.prefix.dev/\ -c https://repo.mamba.pm/conda-forge \ --yes \ "python=${{matrix.python_version}}" pytest exceptiongroup diff --git a/build_mkdocs.sh b/build_mkdocs.sh index f97b0b6..19700a1 100755 --- a/build_mkdocs.sh +++ b/build_mkdocs.sh @@ -18,7 +18,7 @@ if [ ! -d "$WASM_ENV_PREFIX" ]; then echo "Creating wasm env $WASM_ENV_NAME" micromamba create -n $WASM_ENV_NAME \ --platform=emscripten-wasm32 \ - -c https://repo.prefix.dev/emscripten-forge-dev\ + -c https://repo.prefix.dev/emscripten-forge-4x\ -c https://repo.prefix.dev/conda-forge \ --yes \ python=$PYTHON_VERSION "pybind11<3" nlohmann_json pybind11_json numpy \ diff --git a/environment-dev.yml b/environment-dev.yml index bcf1601..3359451 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -2,7 +2,7 @@ name: pyjs-wasm channels: - conda-forge - /Users/thorstenbeier/src/recipes/output - - https://repo.prefix.dev/emscripten-forge-dev + - https://repo.prefix.dev/emscripten-forge-4x dependencies: - cmake From c98b4c8d3d8a04082561e5b9f1a9d67bf8ee1b57 Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Mon, 6 Oct 2025 16:22:10 +0200 Subject: [PATCH 03/13] 4x --- .github/workflows/main.yml | 2 +- environment-dev.yml | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 51bc8aa..8eb6706 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -34,7 +34,7 @@ jobs: environment-name: pyjs-wasm condarc: | channels: - - https://repo.prefix.dev/ + - https://repo.prefix.dev/emscripten-forge-4x - conda-forge diff --git a/environment-dev.yml b/environment-dev.yml index 3359451..a9310b3 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -1,7 +1,6 @@ name: pyjs-wasm channels: - conda-forge - - /Users/thorstenbeier/src/recipes/output - https://repo.prefix.dev/emscripten-forge-4x dependencies: From 553b5f0af40ab87ea7a180a24eebcc0b88e7029b Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Mon, 6 Oct 2025 16:24:29 +0200 Subject: [PATCH 04/13] 4x --- .github/workflows/main.yml | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8eb6706..34fbcc9 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,11 +19,7 @@ jobs: strategy: fail-fast: false - matrix: - include: - - emsdk_ver: "3.1.73" - python_version: "3.13" - pybind11_version: "<3" + steps: - uses: actions/checkout@v2 @@ -51,11 +47,11 @@ jobs: -c https://repo.prefix.dev/emscripten-forge-4x\ -c https://repo.mamba.pm/conda-forge \ --yes \ - python=${{matrix.python_version}} \ - "pybind11${{matrix.pybind11_version}}" \ + python=3.13 \ + "pybind11<3 \ nlohmann_json pybind11_json numpy \ pytest bzip2 sqlite zlib zstd libffi \ - exceptiongroup emscripten-abi==${{matrix.emsdk_ver}} \ + exceptiongroup emscripten-abi>=4 \ openssl liblzma @@ -90,7 +86,7 @@ jobs: -c https://repo.prefix.dev/emscripten-forge-4x\ -c https://repo.mamba.pm/conda-forge \ --yes \ - "python=${{matrix.python_version}}" pytest numpy exceptiongroup + "python=3.13" pytest numpy exceptiongroup - name: Test in browser-main @@ -100,7 +96,7 @@ jobs: browser-main \ --conda-env $MAMBA_ROOT_PREFIX/envs/pyjs-build-wasm-with-numpy \ --mount $(pwd)/tests:/tests \ - --mount $(pwd)/module/pyjs:/lib/python${{matrix.python_version}}/site-packages/pyjs \ + --mount $(pwd)/module/pyjs:/lib/python3.13/site-packages/pyjs \ --script main.py \ --work-dir /tests \ --pyjs-dir $(pwd)/build \ @@ -116,7 +112,7 @@ jobs: browser-worker \ --conda-env $MAMBA_ROOT_PREFIX/envs/pyjs-build-wasm-with-numpy \ --mount $(pwd)/tests:/tests \ - --mount $(pwd)/module/pyjs:/lib/python${{matrix.python_version}}/site-packages/pyjs \ + --mount $(pwd)/module/pyjs:/lib/python3.13/site-packages/pyjs \ --script main.py \ --work-dir /tests \ --pyjs-dir $(pwd)/build \ @@ -131,7 +127,7 @@ jobs: -c https://repo.prefix.dev/\ -c https://repo.mamba.pm/conda-forge \ --yes \ - "python=${{matrix.python_version}}" pytest exceptiongroup + "python=3.13" pytest exceptiongroup - name: Test in browser-main-no-numpy run: | @@ -142,7 +138,7 @@ jobs: browser-main \ --conda-env $MAMBA_ROOT_PREFIX/envs/pyjs-build-wasm-no-numpy \ --mount $(pwd)/tests:/tests \ - --mount $(pwd)/module/pyjs:/lib/python${{matrix.python_version}}/site-packages/pyjs \ + --mount $(pwd)/module/pyjs:/lib/python3.13/site-packages/pyjs \ --script main.py \ --work-dir /tests \ --pyjs-dir $(pwd)/build \ From 5648b7e027570a432028facead67a5b3deaecca7 Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Mon, 6 Oct 2025 16:28:54 +0200 Subject: [PATCH 05/13] 4x --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 34fbcc9..635d11a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -47,8 +47,8 @@ jobs: -c https://repo.prefix.dev/emscripten-forge-4x\ -c https://repo.mamba.pm/conda-forge \ --yes \ - python=3.13 \ - "pybind11<3 \ + "python=3.13" \ + "pybind11<3" \ nlohmann_json pybind11_json numpy \ pytest bzip2 sqlite zlib zstd libffi \ exceptiongroup emscripten-abi>=4 \ From 95834a6a98a42384ec2113c1d8904d63e76b0e3d Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Tue, 7 Oct 2025 09:21:39 +0200 Subject: [PATCH 06/13] build --- src/js_timestamp.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/js_timestamp.cpp b/src/js_timestamp.cpp index b6759b5..723281c 100644 --- a/src/js_timestamp.cpp +++ b/src/js_timestamp.cpp @@ -1 +1 @@ -#define PYJS_JS_UTC_TIMESTAMP "2025-09-23 12:30:08.785762" \ No newline at end of file +#define PYJS_JS_UTC_TIMESTAMP "2025-10-07 07:20:42.296449" \ No newline at end of file From 18cc2ce3ac04c0e3d4b2a8b015a6ec1f70154a27 Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Tue, 7 Oct 2025 09:27:28 +0200 Subject: [PATCH 07/13] trigger From e6297f3579d969636b61d8b51be1cbb952f1286a Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Tue, 7 Oct 2025 09:34:20 +0200 Subject: [PATCH 08/13] empackl --- environment-dev.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment-dev.yml b/environment-dev.yml index a9310b3..ef42e07 100644 --- a/environment-dev.yml +++ b/environment-dev.yml @@ -20,7 +20,7 @@ dependencies: - mkdocstrings - mkdocstrings-python - mkdocs-material - - empack >=3.2.0 + - empack >=6.0.0 - jupyter_server # to enable contents - jupyterlite - jupyterlite-xeus >= 3.1.8 From 8ba5530cb7455081a934b109a48e80ba35e367c0 Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Tue, 7 Oct 2025 09:35:22 +0200 Subject: [PATCH 09/13] empackl --- .github/workflows/docs.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 49bf1c3..3dc945a 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -3,7 +3,9 @@ name: docs on: workflow_dispatch: push: - pull_request: + branches: + - main + # pull_request: defaults: From 97684007ddb06c9992251e26a3663792f5d05295 Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Tue, 7 Oct 2025 10:55:53 +0200 Subject: [PATCH 10/13] wip --- include/pyjs/pre_js/init.js | 21 ++++++++++----------- src/convert.cpp | 5 +++-- src/export_js_module.cpp | 30 +++++++++++++++--------------- src/export_py_object.cpp | 4 ++++ src/js_timestamp.cpp | 2 +- tests/atests/async_tests.py | 9 ++++++--- tests/main.py | 9 +++++++++ 7 files changed, 48 insertions(+), 32 deletions(-) diff --git a/include/pyjs/pre_js/init.js b/include/pyjs/pre_js/init.js index 5eabd4d..c59758c 100644 --- a/include/pyjs/pre_js/init.js +++ b/include/pyjs/pre_js/init.js @@ -57,20 +57,18 @@ Module['init_phase_1'] = async function(prefix, python_version, verbose) { - default_scope = Module["globals"]() - // Module["default_scope"] = default_scope; - // Module['_py_objects'].push(Module["default_scope"]); - + var default_scope = Module["main_scope"]() + Module["default_scope"] = default_scope; Module['_py_objects'].push(Module["_interpreter"]); - Module['exec'] = function(code, globals=default_scope) { + Module['exec'] = function(code, globals=default_scope, locals=default_scope) { if(globals === undefined){ globals = Module["globals"]() } - let ret = Module._exec(code, globals) + let ret = Module._exec(code, globals, locals) // console.error("exec done"); if (ret.has_err) { throw ret @@ -79,11 +77,11 @@ Module['init_phase_1'] = async function(prefix, python_version, verbose) { - Module['eval'] = function(code, globals=default_scope) { + Module['eval'] = function(code, globals=default_scope, locals=default_scope) { if(globals === undefined){ globals = Module["globals"]() } - let ret = Module._eval(code, globals) + let ret = Module._eval(code, globals, locals) if (ret.has_err) { throw ret } else { @@ -91,11 +89,11 @@ Module['init_phase_1'] = async function(prefix, python_version, verbose) { } }; - Module['eval_file'] = function(file, globals=default_scope) { + Module['eval_file'] = function(file, globals=default_scope, locals=default_scope) { if(globals === undefined){ globals = Module["globals"]() } - let ret = Module._eval_file(file, globals) + let ret = Module._eval_file(file, globals, locals) if (ret.has_err) { throw ret } @@ -160,7 +158,8 @@ Module['init_phase_1'] = async function(prefix, python_version, verbose) { let types = keys.map(Module['_get_type_string']) - let ret = this._raw_getitem(keys, types, BigInt(keys.length)) + console.log("getitem keys", keys, types); + let ret = this._raw_getitem(keys, types, keys.length) if (ret.has_err) { throw ret } else { diff --git a/src/convert.cpp b/src/convert.cpp index 86c2fcd..92e3cf6 100644 --- a/src/convert.cpp +++ b/src/convert.cpp @@ -15,14 +15,15 @@ namespace pyjs std::pair implicit_py_to_js(py::object& py_ret) { // py::module_ pyjs = py::module_::import("pyjs_utils"); - // const std::string info = pyjs.attr("implicit_convert_info")(py_ret).cast(); + // const std::string info = pyjs.attr("implicit_convert_info")(py_ret).cast(); const std::string info = py_ret.get_type().attr("__name__").str(); if (info == "int") { - return std::make_pair(em::val(py_ret.cast()),false); + auto emval = em::val(py_ret.cast()); + return std::make_pair(std::move(emval),false); } else if (info == "str") { diff --git a/src/export_js_module.cpp b/src/export_js_module.cpp index f5522c2..95cefea 100644 --- a/src/export_js_module.cpp +++ b/src/export_js_module.cpp @@ -22,13 +22,13 @@ namespace pyjs namespace py = pybind11; - em::val eval(const std::string & code, py::object & globals) + em::val eval(const std::string & code, py::object & globals , py::object & locals) { em::val ret = em::val::object();; ret.set("has_err", false); try { - py::object py_ret = py::eval(code, globals); + py::object py_ret = py::eval(code, globals, locals); auto [jsval, is_proxy] = implicit_py_to_js(py_ret); ret.set("ret",jsval); ret.set("is_proxy",is_proxy); @@ -45,13 +45,13 @@ namespace pyjs - em::val exec(const std::string & code, py::object & globals) + em::val exec(const std::string & code, py::object & globals, py::object & locals) { em::val ret = em::val::object(); ret.set("has_err", false); try { - py::exec(code, globals); + py::exec(code, globals, locals); return ret; } catch (py::error_already_set& e) @@ -65,12 +65,12 @@ namespace pyjs } - em::val eval_file(const std::string & filename, py::object & globals) + em::val eval_file(const std::string & filename, py::object & globals, py::object & locals) { em::val ret = em::val::object(); try { - py::eval_file(filename, globals); + py::eval_file(filename, globals, locals); ret.set("has_err",em::val(false)); return ret; } @@ -138,15 +138,15 @@ namespace pyjs export_py_object(); // // main scope - // em::function("main_scope",em::select_overload( - // []()->py::object{ - // std::cout<<"get scope"<( + []()->py::object{ + std::cout<<"get scope"<py::object{ diff --git a/src/export_py_object.cpp b/src/export_py_object.cpp index 91cea56..5537049 100644 --- a/src/export_py_object.cpp +++ b/src/export_py_object.cpp @@ -89,6 +89,8 @@ namespace pyjs // implicit to py + std::cout<<"getitem n_keys "<( [](py::object& pyobject) -> em::val { + std::cout<<"call 0-ary"<( [](py::object& pyobject, em::val arg1) -> em::val { + std::cout<<"call 1-ary"< Date: Tue, 7 Oct 2025 11:36:22 +0200 Subject: [PATCH 11/13] tests pass --- include/pyjs/pre_js/dynload/dynload.js | 55 ++++++-------------------- include/pyjs/pre_js/init.js | 2 - src/export_py_object.cpp | 3 -- src/js_timestamp.cpp | 2 +- tests/atests/async_tests.py | 5 +-- 5 files changed, 15 insertions(+), 52 deletions(-) diff --git a/include/pyjs/pre_js/dynload/dynload.js b/include/pyjs/pre_js/dynload/dynload.js index 34426f1..bd9b341 100644 --- a/include/pyjs/pre_js/dynload/dynload.js +++ b/include/pyjs/pre_js/dynload/dynload.js @@ -20,7 +20,6 @@ function createLock() { // * global // * undefined // when we dont function libraryType (path) { - console.log("check lib type for", path); if (path.includes("cpython-3") && path.includes("-wasm32-emscripten.so")) { return "local"; } @@ -40,58 +39,38 @@ async function loadDynlibsFromPackage( dynlibPaths, ) { - - - // if(pkg_file_name == "zlib") { - // console.log("zlib is always loaded, skipping loading from package"); - // return; - // } if(prefix != "/") { throw `only root prefix / is supported for loading shared libraries from packages, got: ${prefix}`; } - console.log("LAPALUZA"); - // console.log("load shared objects from package:", pkg_file_name, dynlibPaths); for (let i = 0; i < dynlibPaths.length; i++) { let path = dynlibPaths[i]; - // // if the name contains "_tests" we skip loading the shared libraries - // if (path.includes("_tests") || path.includes("_simd")) { - // console.log(`skipping loading shared libraries from package ${pkg_file_name} because it is a test package or simd package`); - // continue; - // } - const isSymlink = FS.isLink(FS.lstat(path)); if (isSymlink) { - console.log(`skipping symlinked shared library ${path} from package ${pkg_file_name}`); continue; } const releaseDynlibLock = await _lock(); try { - //#define RTLD_LAZY 1 - // #define RTLD_NOW 2 - // #define RTLD_NOLOAD 4 - // #define RTLD_NODELETE 4096 - // #define RTLD_GLOBAL 256 - // #define RTLD_LOCAL 0 + // RTLD_LAZY 1 + // RTLD_NOW 2 + // RTLD_NOLOAD 4 + // RTLD_NODELETE 4096 + // RTLD_GLOBAL 256 + // RTLD_LOCAL 0 const libt = libraryType(path); let flag = 2; // RTLD_NOW - // if(libt === "local"){ - // console.log(`loading local shared library ${path} from package ${pkg_file_name}`); - // flag = 2 /* RTLD_NOW */ | 0 /* RTLD_LOCAL */; - // } - // else if(libt === "global"){ - // console.log(`loading global shared library ${path} from package ${pkg_file_name}`); - // flag = 2 /* RTLD_NOW */ | 256 /* RTLD_GLOBAL */; - // } - // else{ - // console.log(`loading shared library ${path} from package ${pkg_file_name} with default flags`); - // } + if(libt === "local"){ + flag = 2 /* RTLD_NOW */ | 0 /* RTLD_LOCAL */; + } + else if(libt === "global"){ + flag = 2 /* RTLD_NOW */ | 256 /* RTLD_GLOBAL */; + } const stack = Module.stackSave(); const pathUTF8 = Module.stringToUTF8OnStack(path); @@ -129,15 +108,7 @@ async function loadDynlibsFromPackage( } - if(libt !== "local"){ - console.log('re-register shared library with filename without path'); - // get filename from path - const filename = getFilenameFromPath(path); - console.log(`registering shared library ${path} from package ${pkg_file_name} as ${filename} in global namespace`); - // Module.LDSO.loadedLibsByName[filename] = Module.LDSO.loadedLibsByName[path]; - - } - console.log(`!!!!!!!!!shared library ${path} from package ${pkg_file_name} is ready`); + } catch (e) { throw e; } finally { diff --git a/include/pyjs/pre_js/init.js b/include/pyjs/pre_js/init.js index c59758c..1651d1b 100644 --- a/include/pyjs/pre_js/init.js +++ b/include/pyjs/pre_js/init.js @@ -156,9 +156,7 @@ Module['init_phase_1'] = async function(prefix, python_version, verbose) { Module['pyobject'].prototype.get = function(...keys) { - let types = keys.map(Module['_get_type_string']) - console.log("getitem keys", keys, types); let ret = this._raw_getitem(keys, types, keys.length) if (ret.has_err) { throw ret diff --git a/src/export_py_object.cpp b/src/export_py_object.cpp index 5537049..b6e3a1a 100644 --- a/src/export_py_object.cpp +++ b/src/export_py_object.cpp @@ -88,9 +88,6 @@ namespace pyjs // implicit to py - - std::cout<<"getitem n_keys "< Date: Tue, 7 Oct 2025 11:42:09 +0200 Subject: [PATCH 12/13] tests pass --- include/pyjs/pre_js/load_pkg.js | 5 ----- 1 file changed, 5 deletions(-) diff --git a/include/pyjs/pre_js/load_pkg.js b/include/pyjs/pre_js/load_pkg.js index 093685e..1687af3 100644 --- a/include/pyjs/pre_js/load_pkg.js +++ b/include/pyjs/pre_js/load_pkg.js @@ -19,10 +19,8 @@ Module["mkdirs"] = function (dirname) { Module["_untar_from_python"] = function(tarball_path, target_dir = "") { - Module.exec("print('lalalala')"); Module.exec(` def _py_untar(tarball_path, target_dir): - print("1") import tarfile import json from pathlib import Path @@ -30,14 +28,12 @@ def _py_untar(tarball_path, target_dir): import shutil import os import sys - print("2") def check_wasm_magic_number(file_path: Path) -> bool: WASM_BINARY_MAGIC = b"\\0asm" with file_path.open(mode="rb") as file: return file.read(4) == WASM_BINARY_MAGIC - print("3") target_dir = target_dir if target_dir == "": @@ -66,7 +62,6 @@ def _py_untar(tarball_path, target_dir): raise e return s `) - console.log("calling into _py_untar") let shared_libs = Module.eval(`_py_untar("${tarball_path}", "${target_dir}")`) return JSON.parse(shared_libs) From 45ae83603c16a2b57d9ce02ae15a65f5ad22429e Mon Sep 17 00:00:00 2001 From: DerThorsten Date: Tue, 7 Oct 2025 11:50:31 +0200 Subject: [PATCH 13/13] tests pass --- .github/workflows/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 635d11a..8fe8ef3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -44,7 +44,7 @@ jobs: micromamba create -n pyjs-build-wasm \ --platform=emscripten-wasm32 \ - -c https://repo.prefix.dev/emscripten-forge-4x\ + -c https://repo.prefix.dev/emscripten-forge-4x \ -c https://repo.mamba.pm/conda-forge \ --yes \ "python=3.13" \ @@ -83,7 +83,7 @@ jobs: micromamba activate pyjs-wasm micromamba create -n pyjs-build-wasm-with-numpy \ --platform=emscripten-wasm32 \ - -c https://repo.prefix.dev/emscripten-forge-4x\ + -c https://repo.prefix.dev/emscripten-forge-4x \ -c https://repo.mamba.pm/conda-forge \ --yes \ "python=3.13" pytest numpy exceptiongroup @@ -124,7 +124,7 @@ jobs: micromamba activate pyjs-wasm micromamba create -n pyjs-build-wasm-no-numpy \ --platform=emscripten-wasm32 \ - -c https://repo.prefix.dev/\ + -c https://repo.prefix.dev/emscripten-forge-4x \ -c https://repo.mamba.pm/conda-forge \ --yes \ "python=3.13" pytest exceptiongroup