From c18d916ab970e24e6cabd2cacd9a175fd657f53b Mon Sep 17 00:00:00 2001 From: pmp-p Date: Mon, 28 Jul 2025 13:36:37 +0200 Subject: [PATCH 01/11] docker builder, wasm->libpglite(wasi)->pglite all in one --- docker_rc.sh | 51 +++++++++++++++++++++++ wasm-builder/Dockerfile | 24 +++++++++++ wasm-builder/build-with-docker.sh | 69 +++++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+) create mode 100644 docker_rc.sh create mode 100644 wasm-builder/Dockerfile create mode 100755 wasm-builder/build-with-docker.sh diff --git a/docker_rc.sh b/docker_rc.sh new file mode 100644 index 0000000000000..0832d6be2ef5c --- /dev/null +++ b/docker_rc.sh @@ -0,0 +1,51 @@ +#!/bin/bash + +WORKSPACE=$(pwd) + +echo " + +SDK prepare + +" + +pushd / + # this is minimum required to **use** sdk on docker/debian 12, not build it + apt-get update && apt-get --yes install git wget curl lz4 xz-utils bison flex pkg-config autoconf make + + if [ -f $WORKSPACE/sdk.tar.lz4 ] + then + tar xf $WORKSPACE/sdk.tar.lz4 --use-compress-program=lz4 + fi + + if [ -d $SDKROOT/wasisdk/upstream ] + then + echo "wasi sdk common support is already installed" + else + tar xf ${WORKSPACE}/prebuilt/wasi-sdk-25.tar.xz + fi + + if [ -d $SDKROOT/wasisdk/upstream/lib ] + then + echo "wasi sdk $(arch) support is already installed" + else + pushd $SDKROOT/wasisdk + if arch|grep -q aarch64 + then + wget https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-25/wasi-sdk-25.0-arm64-linux.tar.gz -O/tmp/sdk.tar.gz + else + wget https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-25/wasi-sdk-25.0-x86_64-linux.tar.gz -O/tmp/sdk.tar.gz + fi + tar xfz /tmp/sdk.tar.gz && rm /tmp/sdk.tar.gz + mv wasi-sdk-25.0-*/{bin,lib} upstream/ + popd + fi + +popd + +echo " + +Setting up SDK shell + +" + +${SDKROOT}/wasm32-bi-emscripten-shell.sh diff --git a/wasm-builder/Dockerfile b/wasm-builder/Dockerfile new file mode 100644 index 0000000000000..3564af11157d8 --- /dev/null +++ b/wasm-builder/Dockerfile @@ -0,0 +1,24 @@ +FROM debian:12 AS build_sdk + +ARG PG_VERSION=17.5 +ARG SDK_VERSION=3.1.74.11bi-w-n +ARG DEBUG=false +ARG OBJDUMP=true +ARG PG_BRANCH=REL_17_5_WASM-pglite + +ENV \ + PG_VERSION=$PG_VERSION \ + SDK_VERSION=$SDK_VERSION \ + SDKROOT=/tmp/sdk \ + SYS_PYTHON=/usr/bin/python3 \ + DEBUG=$DEBUG \ + BUILDS=3.12 \ + EMFLAVOUR=3.1.74 + +RUN apt-get update && \ + apt-get install -y git wget lz4 bzip2 pv bash python3 build-essential libreadline-dev zlib1g-dev bison flex xz-utils perl + +RUN apt-get install -y clang autoconf wget curl lz4 lsb-release zlib1g-dev libssl-dev + +# RUN bash -c "wget -qO- https://github.com/electric-sql/portable-sdk/releases/download/3.1.74.11bi-w-n/python3.13-wasm-sdk-debian12-$(arch).tar.lz4 | lz4 -d | tar -xvf -" +RUN bash -c "wget -qO- https://github.com/electric-sql/portable-sdk/releases/download/3.1.74.11.11/python3.13-wasm-sdk-debian12-$(arch).tar.lz4 | lz4 -d | tar -xvf -" \ No newline at end of file diff --git a/wasm-builder/build-with-docker.sh b/wasm-builder/build-with-docker.sh new file mode 100755 index 0000000000000..6396781c40027 --- /dev/null +++ b/wasm-builder/build-with-docker.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# we are using a custom emsdk to build pglite wasm +# this is available as a docker image under electricsql/pglite-builder +IMG_NAME=${IMG_NAME:-"electricsql/pglite-builder"} + +[ -f postgres-pglite/configure ] || ln -s . postgres-pglite + +export WORKSPACE=${GITHUB_WORKSPACE:-$(pwd)} + +# normally would default to /workspace but that may cause trouble with debug paths in some IDE +export DOCKER_WORKSPACE=${DOCKER_WORKSPACE:-$WORKSPACE} + +cd $(realpath ${WORKSPACE}/postgres-pglite) + +[ -f ${BUILD_CONFIG:-postgres-pglite}/.buildconfig ] && cp ${BUILD_CONFIG:-postgres-pglite}/.buildconfig .buildconfig +[ -f ./pglite/.buildconfig ] && cp ./pglite/.buildconfig .buildconfig + + +source .buildconfig + +cat .buildconfig + + +if echo $IMG_NAME|grep -q debian +then + IMG_NAME="debian" + IMG_TAG="12" + wget -q -Osdk.tar.lz4 \ + https://github.com/electric-sql/portable-sdk/releases/download/${SDK_VERSION}/python3.13-wasm-sdk-${IMG_NAME}${IMG_TAG}-$(arch).tar.lz4 +else + IMG_TAG="${PG_VERSION}_${SDK_VERSION}" +fi + + +mkdir -p dist/pglite dist/extensions-emsdk + +if echo -n $@|grep -q it$ +then + PROMPT="&& bash ) || bash" +else + PROMPT=")" +fi + +if $WASI +then + OUT_DIR=wasi +else + OUT_DIR=emscripten +fi + +if $DEBUG +then + MAP_OUT_DIRS="-v ${WORKSPACE}/postgres-pglite/out/${OUT_DIR}/build:/tmp/sdk/build:rw -v ${WORKSPACE}/postgres-pglite/out/${OUT_DIR}/pglite:/tmp/pglite:rw" +else + MAP_OUT_DIRS="" +fi + +docker run $@ \ + --rm \ + --env-file .buildconfig \ + -e DEBUG=${DEBUG:-false} \ + -e WASI=${WASI:-false} \ + --workdir=${DOCKER_WORKSPACE} \ + -v ${WORKSPACE}/postgres-pglite:${DOCKER_WORKSPACE}:rw \ + -v ${WORKSPACE}/postgres-pglite/dist:/tmp/sdk/dist:rw \ + $MAP_OUT_DIRS \ + $IMG_NAME:$IMG_TAG \ + bash --noprofile --rcfile ./docker_rc.sh -ci "( ./wasm-build.sh ${WHAT:-\"contrib extra\"} $PROMPT" From 4875f3f069f4bd5ef48e6d54d8bb01520c97a36e Mon Sep 17 00:00:00 2001 From: Tudor Zaharia <37596026+tdrz@users.noreply.github.com> Date: Mon, 28 Jul 2025 16:34:37 +0200 Subject: [PATCH 02/11] Update Dockerfile --- wasm-builder/Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/wasm-builder/Dockerfile b/wasm-builder/Dockerfile index 3564af11157d8..a806e5e098b16 100644 --- a/wasm-builder/Dockerfile +++ b/wasm-builder/Dockerfile @@ -20,5 +20,4 @@ RUN apt-get update && \ RUN apt-get install -y clang autoconf wget curl lz4 lsb-release zlib1g-dev libssl-dev -# RUN bash -c "wget -qO- https://github.com/electric-sql/portable-sdk/releases/download/3.1.74.11bi-w-n/python3.13-wasm-sdk-debian12-$(arch).tar.lz4 | lz4 -d | tar -xvf -" -RUN bash -c "wget -qO- https://github.com/electric-sql/portable-sdk/releases/download/3.1.74.11.11/python3.13-wasm-sdk-debian12-$(arch).tar.lz4 | lz4 -d | tar -xvf -" \ No newline at end of file +RUN bash -c "wget -qO- https://github.com/electric-sql/portable-sdk/releases/download/3.1.74.12.0/python3.13-wasm-sdk-debian12-$(arch).tar.lz4 | lz4 -d | tar -xvf -" From 80621caed5773300a7caf1f88bc2573b1861840b Mon Sep 17 00:00:00 2001 From: pmp-p Date: Thu, 31 Jul 2025 13:18:07 +0200 Subject: [PATCH 03/11] sdk hotfix --- docker_rc.sh | 11 + hotfix/library_dylink.js | 1288 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 1299 insertions(+) create mode 100644 hotfix/library_dylink.js diff --git a/docker_rc.sh b/docker_rc.sh index 0832d6be2ef5c..bd9c42f40d1d5 100644 --- a/docker_rc.sh +++ b/docker_rc.sh @@ -46,6 +46,17 @@ echo " Setting up SDK shell + +Applying Hotfixes +------------------------------------------------------------------------------------------ " +echo "linker fix" +cp -vf hotfix/library_dylink.js ${SDKROOT}/emsdk/upstream/emscripten/src/ + + + +echo "------------------------------------------------------------------------------------------" + + ${SDKROOT}/wasm32-bi-emscripten-shell.sh diff --git a/hotfix/library_dylink.js b/hotfix/library_dylink.js new file mode 100644 index 0000000000000..8a44339b30670 --- /dev/null +++ b/hotfix/library_dylink.js @@ -0,0 +1,1288 @@ +/** + * @license + * Copyright 2020 The Emscripten Authors + * SPDX-License-Identifier: MIT + * + * Dynamic library loading + */ + +#if !RELOCATABLE +#error "library_dylink.js requires RELOCATABLE" +#endif + +var LibraryDylink = { +#if FILESYSTEM + $registerWasmPlugin__deps: ['$preloadPlugins'], + $registerWasmPlugin: () => { + // Use string keys here to avoid minification since the plugin consumer + // also uses string keys. + var wasmPlugin = { + 'promiseChainEnd': Promise.resolve(), + 'canHandle': (name) => { + return !Module['noWasmDecoding'] && name.endsWith('.so') + }, + 'handle': (byteArray, name, onload, onerror) => { + // loadWebAssemblyModule can not load modules out-of-order, so rather + // than just running the promises in parallel, this makes a chain of + // promises to run in series. + wasmPlugin['promiseChainEnd'] = wasmPlugin['promiseChainEnd'].then( + () => loadWebAssemblyModule(byteArray, {loadAsync: true, nodelete: true}, name, {})).then( + (exports) => { +#if DYLINK_DEBUG + dbg(`registering preloadedWasm: ${name}`); +#endif + preloadedWasm[name] = exports; + onload(byteArray); + }, + (error) => { + err(`failed to instantiate wasm: ${name}: ${error}`); + onerror(); + }); + } + }; + preloadPlugins.push(wasmPlugin); + }, + + $preloadedWasm__deps: ['$registerWasmPlugin'], + $preloadedWasm__postset: ` + registerWasmPlugin(); + `, + $preloadedWasm: {}, +#endif // FILESYSTEM + + $isSymbolDefined: (symName) => { + // Ignore 'stub' symbols that are auto-generated as part of the original + // `wasmImports` used to instantiate the main module. + var existing = wasmImports[symName]; + if (!existing || existing.stub) { + return false; + } +#if ASYNCIFY + // Even if a symbol exists in wasmImports, and is not itself a stub, it + // could be an ASYNCIFY wrapper function that wraps a stub function. + if (symName in asyncifyStubs && !asyncifyStubs[symName]) { + return false; + } +#endif + return true; + }, + + // Dynamic version of shared.py:make_invoke. This is needed for invokes + // that originate from side modules since these are not known at JS + // generation time. +#if !DISABLE_EXCEPTION_CATCHING || SUPPORT_LONGJMP == 'emscripten' + $createInvokeFunction__internal: true, + $createInvokeFunction__deps: ['$dynCall', 'setThrew', '$stackSave', '$stackRestore'], + $createInvokeFunction: (sig) => (ptr, ...args) => { + var sp = stackSave(); + try { + return dynCall(sig, ptr, args); + } catch(e) { + stackRestore(sp); + // Create a try-catch guard that rethrows the Emscripten EH exception. +#if EXCEPTION_STACK_TRACES + // Exceptions thrown from C++ and longjmps will be an instance of + // EmscriptenEH. + if (!(e instanceof EmscriptenEH)) throw e; +#else + // Exceptions thrown from C++ will be a pointer (number) and longjmp + // will throw the number Infinity. Use the compact and fast "e !== e+0" + // test to check if e was not a Number. + if (e !== e+0) throw e; +#endif + _setThrew(1, 0); +#if WASM_BIGINT + // In theory this if statement could be done on + // creating the function, but I just added this to + // save wasting code space as it only happens on exception. + if (sig[0] == "j") return 0n; +#endif + } + }, +#endif + + // Resolve a global symbol by name. This is used during module loading to + // resolve imports, and by `dlsym` when used with `RTLD_DEFAULT`. + // Returns both the resolved symbol (i.e. a function or a global) along with + // the canonical name of the symbol (in some cases is modify the symbol as + // part of the loop process, so that actual symbol looked up has a different + // name). + $resolveGlobalSymbol__deps: ['$isSymbolDefined', +#if !DISABLE_EXCEPTION_CATCHING || SUPPORT_LONGJMP == 'emscripten' + '$createInvokeFunction', +#endif + ], + $resolveGlobalSymbol__internal: true, + $resolveGlobalSymbol: (symName, direct = false) => { + var sym; +#if !WASM_BIGINT + // First look for the orig$ symbol which is the symbol without i64 + // legalization performed. + if (direct && ('orig$' + symName in wasmImports)) { + symName = 'orig$' + symName; + } +#endif + if (isSymbolDefined(symName)) { + sym = wasmImports[symName]; + } +#if !DISABLE_EXCEPTION_CATCHING || SUPPORT_LONGJMP == 'emscripten' + // Asm.js-style exception handling: invoke wrapper generation + else if (symName.startsWith('invoke_')) { + // Create (and cache) new invoke_ functions on demand. + sym = wasmImports[symName] = createInvokeFunction(symName.split('_')[1]); + } +#endif +#if !DISABLE_EXCEPTION_CATCHING + else if (symName.startsWith('__cxa_find_matching_catch_')) { + // When the main module is linked we create whichever variants of + // `__cxa_find_matching_catch_` (see jsifier.js) that we know are needed, + // but a side module loaded at runtime might need different/additional + // variants so we create those dynamically. + sym = wasmImports[symName] = (...args) => { +#if MEMORY64 + args = args.map(Number); +#endif + var rtn = findMatchingCatch(args); + return {{{ to64('rtn') }}}; + } + } +#endif + return {sym, name: symName}; + }, + + $GOT: {}, + $currentModuleWeakSymbols: '=new Set({{{ JSON.stringify(Array.from(WEAK_IMPORTS)) }}})', + + // Create globals to each imported symbol. These are all initialized to zero + // and get assigned later in `updateGOT` + $GOTHandler__internal: true, + $GOTHandler__deps: ['$GOT', '$currentModuleWeakSymbols'], + $GOTHandler: { + get(obj, symName) { + var rtn = GOT[symName]; + if (!rtn) { + rtn = GOT[symName] = new WebAssembly.Global({'value': '{{{ POINTER_WASM_TYPE }}}', 'mutable': true}); +#if DYLINK_DEBUG == 2 + dbg("new GOT entry: " + symName); +#endif + } + if (!currentModuleWeakSymbols.has(symName)) { + // Any non-weak reference to a symbol marks it as `required`, which + // enabled `reportUndefinedSymbols` to report undefeind symbol errors + // correctly. + rtn.required = true; + } + return rtn; + } + }, + + $isInternalSym__internal: true, + $isInternalSym: (symName) => { + // TODO: find a way to mark these in the binary or avoid exporting them. + return [ + '__cpp_exception', + '__c_longjmp', + '__wasm_apply_data_relocs', + '__dso_handle', + '__tls_size', + '__tls_align', + '__set_stack_limits', + '_emscripten_tls_init', + '__wasm_init_tls', + '__wasm_call_ctors', + '__start_em_asm', + '__stop_em_asm', + '__start_em_js', + '__stop_em_js', + ].includes(symName) || symName.startsWith('__em_js__') +#if SPLIT_MODULE + // Exports synthesized by wasm-split should be prefixed with '%' + || symName[0] == '%' +#endif + ; + }, + + $updateGOT__internal: true, + $updateGOT__deps: ['$GOT', '$isInternalSym', '$addFunction', '$getFunctionAddress'], + $updateGOT: (exports, replace) => { +#if DYLINK_DEBUG + dbg("updateGOT: adding " + Object.keys(exports).length + " symbols"); +#endif + for (var symName in exports) { + if (isInternalSym(symName)) { + continue; + } + + var value = exports[symName]; +#if !WASM_BIGINT + if (symName.startsWith('orig$')) { + symName = symName.split('$')[1]; + replace = true; + } +#endif + + GOT[symName] ||= new WebAssembly.Global({'value': '{{{ POINTER_WASM_TYPE }}}', 'mutable': true}); + if (replace || GOT[symName].value == 0) { +#if DYLINK_DEBUG == 2 + dbg(`updateGOT: before: ${symName} : ${GOT[symName].value}`); +#endif + if (typeof value == 'function') { + GOT[symName].value = {{{ to64('addFunction(value)') }}}; +#if DYLINK_DEBUG == 2 + dbg(`updateGOT: FUNC: ${symName} : ${GOT[symName].value}`); +#endif + } else if (typeof value == {{{ POINTER_JS_TYPE }}}) { + GOT[symName].value = value; + } else { + err(`unhandled export type for '${symName}': ${typeof value}`); + } +#if DYLINK_DEBUG == 2 + dbg(`updateGOT: after: ${symName} : ${GOT[symName].value} (${value})`); +#endif + } +#if DYLINK_DEBUG + else if (GOT[symName].value != value) { + dbg(`updateGOT: EXISTING SYMBOL: ${symName} : ${GOT[symName].value} (${value})`); + } +#endif + } +#if DYLINK_DEBUG + dbg("done updateGOT"); +#endif + }, + + // Applies relocations to exported things. + $relocateExports__internal: true, + $relocateExports__deps: ['$updateGOT'], + $relocateExports__docs: '/** @param {boolean=} replace */', + $relocateExports: (exports, memoryBase, replace) => { + var relocated = {}; + + for (var e in exports) { + var value = exports[e]; +#if SPLIT_MODULE + // Do not modify exports synthesized by wasm-split + if (e.startsWith('%')) { + relocated[e] = value + continue; + } +#endif + if (typeof value == 'object') { + // a breaking change in the wasm spec, globals are now objects + // https://github.com/WebAssembly/mutable-global/issues/1 + value = value.value; + } + if (typeof value == {{{ POINTER_JS_TYPE }}}) { + value += {{{ to64('memoryBase') }}}; + } + relocated[e] = value; + } + updateGOT(relocated, replace); + return relocated; + }, + + $reportUndefinedSymbols__internal: true, + $reportUndefinedSymbols__deps: ['$GOT', '$resolveGlobalSymbol'], + $reportUndefinedSymbols: () => { +#if DYLINK_DEBUG + dbg('reportUndefinedSymbols'); +#endif + for (var [symName, entry] of Object.entries(GOT)) { + if (entry.value == 0) { + var value = resolveGlobalSymbol(symName, true).sym; + if (!value && !entry.required) { + // Ignore undefined symbols that are imported as weak. +#if DYLINK_DEBUG + dbg(`ignoring undefined weak symbol: ${symName}`); +#endif + continue; + } +#if ASSERTIONS + assert(value, `undefined symbol '${symName}'. perhaps a side module was not linked in? if this global was expected to arrive from a system library, try to build the MAIN_MODULE with EMCC_FORCE_STDLIBS=1 in the environment`); +#endif +#if DYLINK_DEBUG == 2 + dbg(`assigning dynamic symbol from main module: ${symName} -> ${prettyPrint(value)}`); +#endif + if (typeof value == 'function') { + /** @suppress {checkTypes} */ + entry.value = {{{ to64('addFunction(value, value.sig)') }}}; +#if DYLINK_DEBUG == 2 + dbg(`assigning table entry for : ${symName} -> ${entry.value}`); +#endif + } else if (typeof value == 'number') { + entry.value = {{{ to64('value') }}}; +#if MEMORY64 + } else if (typeof value == 'bigint') { + entry.value = value; +#endif + } else { + throw new Error(`bad export type for '${symName}': ${typeof value}`); + } + } + } +#if DYLINK_DEBUG + dbg('done reportUndefinedSymbols'); +#endif + }, + + // dynamic linker/loader (a-la ld.so on ELF systems) + $LDSO__deps: ['$newDSO'], + $LDSO: { + // name -> dso [refcount, name, module, global]; Used by dlopen + loadedLibsByName: {}, + // handle -> dso; Used by dlsym + loadedLibsByHandle: {}, + init() { +#if ASSERTIONS + // This function needs to run after the initial wasmImports object + // as been created. + assert(wasmImports); +#endif + newDSO('__main__', {{{ cDefs.RTLD_DEFAULT }}}, wasmImports); + }, + }, + + $dlSetError__internal: true, + $dlSetError__deps: ['__dl_seterr', '$stringToUTF8OnStack', '$stackSave', '$stackRestore'], + $dlSetError: (msg) => { +#if DYLINK_DEBUG + dbg(`dlSetError: ${msg}`); +#endif + var sp = stackSave(); + var cmsg = stringToUTF8OnStack(msg); + ___dl_seterr(cmsg, 0); + stackRestore(sp); + }, + + // We support some amount of allocation during startup in the case of + // dynamic linking, which needs to allocate memory for dynamic libraries that + // are loaded. That has to happen before the main program can start to run, + // because the main program needs those linked in before it runs (so we can't + // use normally malloc from the main program to do these allocations). + // + // Allocate memory even if malloc isn't ready yet. The allocated memory here + // must be zero initialized since its used for all static data, including bss. + $getMemory__noleakcheck: true, + $getMemory__deps: ['$GOT', '__heap_base', '$alignMemory', 'calloc'], + $getMemory: (size) => { + // After the runtime is initialized, we must only use sbrk() normally. +#if DYLINK_DEBUG + dbg("getMemory: " + size + " runtimeInitialized=" + runtimeInitialized); +#endif + if (runtimeInitialized) { + // Currently we don't support freeing of static data when modules are + // unloaded via dlclose. This function is tagged as `noleakcheck` to + // avoid having this reported as leak. + return _calloc(size, 1); + } + var ret = ___heap_base; + // Keep __heap_base stack aligned. + var end = ret + alignMemory(size, {{{ STACK_ALIGN }}}); +#if ASSERTIONS + assert(end <= HEAP8.length, 'failure to getMemory - memory growth etc. is not supported there, call malloc/sbrk directly or increase INITIAL_MEMORY'); +#endif + ___heap_base = end; + GOT['__heap_base'].value = {{{ to64('end') }}}; + return ret; + }, + + // returns the side module metadata as an object + // { memorySize, memoryAlign, tableSize, tableAlign, neededDynlibs} + $getDylinkMetadata__deps: ['$UTF8ArrayToString'], + $getDylinkMetadata__internal: true, + $getDylinkMetadata: (binary) => { + var offset = 0; + var end = 0; + + function getU8() { + return binary[offset++]; + } + + function getLEB() { + var ret = 0; + var mul = 1; + while (1) { + var byte = binary[offset++]; + ret += ((byte & 0x7f) * mul); + mul *= 0x80; + if (!(byte & 0x80)) break; + } + return ret; + } + + function getString() { + var len = getLEB(); + offset += len; + return UTF8ArrayToString(binary, offset - len, len); + } + + /** @param {string=} message */ + function failIf(condition, message) { + if (condition) throw new Error(message); + } + + var name = 'dylink.0'; + if (binary instanceof WebAssembly.Module) { + var dylinkSection = WebAssembly.Module.customSections(binary, name); + if (dylinkSection.length === 0) { + name = 'dylink' + dylinkSection = WebAssembly.Module.customSections(binary, name); + } + failIf(dylinkSection.length === 0, 'need dylink section'); + binary = new Uint8Array(dylinkSection[0]); + end = binary.length + } else { + var int32View = new Uint32Array(new Uint8Array(binary.subarray(0, 24)).buffer); +#if SUPPORT_BIG_ENDIAN + var magicNumberFound = int32View[0] == 0x6d736100 || int32View[0] == 0x0061736d; +#else + var magicNumberFound = int32View[0] == 0x6d736100; +#endif + failIf(!magicNumberFound, 'need to see wasm magic number'); // \0asm + // we should see the dylink custom section right after the magic number and wasm version + failIf(binary[8] !== 0, 'need the dylink section to be first') + offset = 9; + var section_size = getLEB(); //section size + end = offset + section_size; + name = getString(); + } + + var customSection = { neededDynlibs: [], tlsExports: new Set(), weakImports: new Set() }; + if (name == 'dylink') { + customSection.memorySize = getLEB(); + customSection.memoryAlign = getLEB(); + customSection.tableSize = getLEB(); + customSection.tableAlign = getLEB(); + // shared libraries this module needs. We need to load them first, so that + // current module could resolve its imports. (see tools/shared.py + // WebAssembly.make_shared_library() for "dylink" section extension format) + var neededDynlibsCount = getLEB(); + for (var i = 0; i < neededDynlibsCount; ++i) { + var libname = getString(); + customSection.neededDynlibs.push(libname); + } + } else { + failIf(name !== 'dylink.0'); + var WASM_DYLINK_MEM_INFO = 0x1; + var WASM_DYLINK_NEEDED = 0x2; + var WASM_DYLINK_EXPORT_INFO = 0x3; + var WASM_DYLINK_IMPORT_INFO = 0x4; + var WASM_SYMBOL_TLS = 0x100; + var WASM_SYMBOL_BINDING_MASK = 0x3; + var WASM_SYMBOL_BINDING_WEAK = 0x1; + while (offset < end) { + var subsectionType = getU8(); + var subsectionSize = getLEB(); + if (subsectionType === WASM_DYLINK_MEM_INFO) { + customSection.memorySize = getLEB(); + customSection.memoryAlign = getLEB(); + customSection.tableSize = getLEB(); + customSection.tableAlign = getLEB(); + } else if (subsectionType === WASM_DYLINK_NEEDED) { + var neededDynlibsCount = getLEB(); + for (var i = 0; i < neededDynlibsCount; ++i) { + libname = getString(); + customSection.neededDynlibs.push(libname); + } + } else if (subsectionType === WASM_DYLINK_EXPORT_INFO) { + var count = getLEB(); + while (count--) { + var symname = getString(); + var flags = getLEB(); + if (flags & WASM_SYMBOL_TLS) { + customSection.tlsExports.add(symname); + } + } + } else if (subsectionType === WASM_DYLINK_IMPORT_INFO) { + var count = getLEB(); + while (count--) { + var modname = getString(); + var symname = getString(); + var flags = getLEB(); + if ((flags & WASM_SYMBOL_BINDING_MASK) == WASM_SYMBOL_BINDING_WEAK) { + customSection.weakImports.add(symname); + } + } + } else { +#if ASSERTIONS + err(`unknown dylink.0 subsection: ${subsectionType}`) +#endif + // unknown subsection + offset += subsectionSize; + } + } + } + +#if ASSERTIONS + var tableAlign = Math.pow(2, customSection.tableAlign); + assert(tableAlign === 1, `invalid tableAlign ${tableAlign}`); + assert(offset == end); +#endif + +#if DYLINK_DEBUG + dbg(`dylink needed:${customSection.neededDynlibs}`); +#endif + + return customSection; + }, + +#if DYNCALLS || !WASM_BIGINT + $registerDynCallSymbols: (exports) => { + for (var [sym, exp] of Object.entries(exports)) { + if (sym.startsWith('dynCall_') && !Module.hasOwnProperty(sym)) { + Module[sym] = exp; + } + } + }, +#endif + + // Module.symbols <- libModule.symbols (flags.global handler) + $mergeLibSymbols__deps: ['$isSymbolDefined'], + $mergeLibSymbols: (exports, libName) => { +#if DYNCALLS || !WASM_BIGINT + registerDynCallSymbols(exports); +#endif + // add symbols into global namespace TODO: weak linking etc. + for (var [sym, exp] of Object.entries(exports)) { +#if ASSERTIONS == 2 + if (isSymbolDefined(sym)) { + var curr = wasmImports[sym], next = exp; + // don't warn on functions - might be odr, linkonce_odr, etc. + if (!(typeof curr == 'function' && typeof next == 'function')) { + err(`warning: symbol '${sym}' from '${libName}' already exists (duplicate symbol? or weak linking, which isn't supported yet?)`); // + [curr, ' vs ', next]); + } + } +#endif + + // When RTLD_GLOBAL is enabled, the symbols defined by this shared object + // will be made available for symbol resolution of subsequently loaded + // shared objects. + // + // We should copy the symbols (which include methods and variables) from + // SIDE_MODULE to MAIN_MODULE. + const setImport = (target) => { +#if ASYNCIFY + if (target in asyncifyStubs) { + asyncifyStubs[target] = exp; + } +#endif + if (!isSymbolDefined(target)) { + wasmImports[target] = exp; + } + } + setImport(sym); + +#if !hasExportedSymbol('main') + // Special case for handling of main symbol: If a side module exports + // `main` that also acts a definition for `__main_argc_argv` and vice + // versa. + const main_alias = '__main_argc_argv'; + if (sym == 'main') { + setImport(main_alias) + } + if (sym == main_alias) { + setImport('main') + } +#endif + } + }, + +#if DYLINK_DEBUG + $dumpTable__deps: ['$wasmTable'], + $dumpTable: () => { + var len = wasmTable.length; + for (var i = {{{ toIndexType(0) }}} ; i < len; i++) { + dbg(`table: ${i} : ${wasmTable.get(i)}`); + } + }, +#endif + + // Loads a side module from binary data or compiled Module. Returns the module's exports or a + // promise that resolves to its exports if the loadAsync flag is set. + $loadWebAssemblyModule__docs: ` + /** + * @param {string=} libName + * @param {Object=} localScope + * @param {number=} handle + */`, + $loadWebAssemblyModule__deps: [ + '$loadDynamicLibrary', '$getMemory', + '$relocateExports', '$resolveGlobalSymbol', '$GOTHandler', + '$getDylinkMetadata', '$alignMemory', + '$currentModuleWeakSymbols', + '$updateTableMap', + '$wasmTable', + ], + $loadWebAssemblyModule: (binary, flags, libName, localScope, handle) => { +#if DYLINK_DEBUG + dbg(`loadWebAssemblyModule: ${libName}`); +#endif + var metadata = getDylinkMetadata(binary); + currentModuleWeakSymbols = metadata.weakImports; +#if ASSERTIONS + var originalTable = wasmTable; +#endif + + // loadModule loads the wasm module after all its dependencies have been loaded. + // can be called both sync/async. + function loadModule() { + // The first thread to load a given module needs to allocate the static + // table and memory regions. Later threads re-use the same table region + // and can ignore the memory region (since memory is shared between + // threads already). + // If `handle` is specified than it is assumed that the calling thread has + // exclusive access to it for the duration of this function. See the + // locking in `dynlink.c`. + var firstLoad = !handle || !{{{ makeGetValue('handle', C_STRUCTS.dso.mem_allocated, 'i8') }}}; + if (firstLoad) { + // alignments are powers of 2 + var memAlign = Math.pow(2, metadata.memoryAlign); + // prepare memory + var memoryBase = metadata.memorySize ? alignMemory(getMemory(metadata.memorySize + memAlign), memAlign) : 0; // TODO: add to cleanups + var tableBase = metadata.tableSize ? {{{ from64Expr('wasmTable.length') }}} : 0; + if (handle) { + {{{ makeSetValue('handle', C_STRUCTS.dso.mem_allocated, '1', 'i8') }}}; + {{{ makeSetValue('handle', C_STRUCTS.dso.mem_addr, 'memoryBase', '*') }}}; + {{{ makeSetValue('handle', C_STRUCTS.dso.mem_size, 'metadata.memorySize', 'i32') }}}; + {{{ makeSetValue('handle', C_STRUCTS.dso.table_addr, 'tableBase', '*') }}}; + {{{ makeSetValue('handle', C_STRUCTS.dso.table_size, 'metadata.tableSize', 'i32') }}}; + } + } else { + memoryBase = {{{ makeGetValue('handle', C_STRUCTS.dso.mem_addr, '*') }}}; + tableBase = {{{ makeGetValue('handle', C_STRUCTS.dso.table_addr, '*') }}}; + } + + var tableGrowthNeeded = tableBase + metadata.tableSize - {{{ from64Expr('wasmTable.length') }}}; + if (tableGrowthNeeded > 0) { +#if DYLINK_DEBUG + dbg("loadModule: growing table: " + tableGrowthNeeded); +#endif + wasmTable.grow({{{ toIndexType('tableGrowthNeeded') }}}); + } +#if DYLINK_DEBUG + dbg("loadModule: memory[" + memoryBase + ":" + (memoryBase + metadata.memorySize) + "]" + + " table[" + tableBase + ":" + (tableBase + metadata.tableSize) + "]"); +#endif + + // This is the export map that we ultimately return. We declare it here + // so it can be used within resolveSymbol. We resolve symbols against + // this local symbol map in the case there they are not present on the + // global Module object. We need this fallback because Modules sometime + // need to import their own symbols + var moduleExports; + + function resolveSymbol(sym) { + var resolved = resolveGlobalSymbol(sym).sym; + if (!resolved && localScope) { + resolved = localScope[sym]; + } + if (!resolved) { + resolved = moduleExports[sym]; + } +#if ASSERTIONS + assert(resolved, `undefined symbol '${sym}'. perhaps a side module was not linked in? if this global was expected to arrive from a system library, try to build the MAIN_MODULE with EMCC_FORCE_STDLIBS=1 in the environment`); +#endif + return resolved; + } + + // TODO kill ↓↓↓ (except "symbols local to this module", it will likely be + // not needed if we require that if A wants symbols from B it has to link + // to B explicitly: similarly to -Wl,--no-undefined) + // + // wasm dynamic libraries are pure wasm, so they cannot assist in + // their own loading. When side module A wants to import something + // provided by a side module B that is loaded later, we need to + // add a layer of indirection, but worse, we can't even tell what + // to add the indirection for, without inspecting what A's imports + // are. To do that here, we use a JS proxy (another option would + // be to inspect the binary directly). + var proxyHandler = { + get(stubs, prop) { + // symbols that should be local to this module + switch (prop) { + case '__memory_base': + return {{{ to64('memoryBase') }}}; + case '__table_base': + return {{{ to64('tableBase') }}}; +#if MEMORY64 +#if MEMORY64 == 2 + case '__memory_base32': + return memoryBase; +#endif + case '__table_base32': + return tableBase; +#endif + } + if (prop in wasmImports && !wasmImports[prop].stub) { + // No stub needed, symbol already exists in symbol table + return wasmImports[prop]; + } + // Return a stub function that will resolve the symbol + // when first called. + if (!(prop in stubs)) { + var resolved; + stubs[prop] = (...args) => { + resolved ||= resolveSymbol(prop); + if (!resolved) { +/* + if (prop==='getTempRet0') + return __emscripten_tempret_get(...args); + if (prop==='setTempRet0') + return __emscripten_tempret_set(...args); +*/ + throw new Error(); + } + return resolved(...args); + }; + } + return stubs[prop]; + } + }; + var proxy = new Proxy({}, proxyHandler); + var info = { + 'GOT.mem': new Proxy({}, GOTHandler), + 'GOT.func': new Proxy({}, GOTHandler), + 'env': proxy, + '{{{ WASI_MODULE_NAME }}}': proxy, + }; + + function postInstantiation(module, instance) { +#if ASSERTIONS + // the table should be unchanged + assert(wasmTable === originalTable); +#endif +#if PTHREADS + if (!ENVIRONMENT_IS_PTHREAD && libName) { +#if DYLINK_DEBUG + dbg(`registering sharedModules: ${libName}`) +#endif + // cache all loaded modules in `sharedModules`, which gets passed + // to new workers when they are created. + sharedModules[libName] = module; + } +#endif + // add new entries to functionsInTableMap + updateTableMap(tableBase, metadata.tableSize); + moduleExports = relocateExports(instance.exports, memoryBase); +#if ASYNCIFY + moduleExports = Asyncify.instrumentWasmExports(moduleExports); +#endif + if (!flags.allowUndefined) { + reportUndefinedSymbols(); + } +#if STACK_OVERFLOW_CHECK >= 2 + // If the runtime has already been initialized we set the stack limits + // now. Otherwise this is delayed until `setDylinkStackLimits` is + // called after initialization. + if (moduleExports['__set_stack_limits'] && runtimeInitialized) { + moduleExports['__set_stack_limits']({{{ to64('_emscripten_stack_get_base()') }}}, {{{ to64('_emscripten_stack_get_end()') }}}); + } +#endif + +#if MAIN_MODULE + function addEmAsm(addr, body) { + var args = []; + var arity = 0; + for (; arity < 16; arity++) { + if (body.indexOf('$' + arity) != -1) { + args.push('$' + arity); + } else { + break; + } + } + args = args.join(','); + var func = `(${args}) => { ${body} };`; +#if DYLINK_DEBUG + dbg(`adding new EM_ASM constant at: ${ptrToString(start)}`); +#endif + {{{ makeEval('ASM_CONSTS[start] = eval(func)') }}}; + } + + // Add any EM_ASM function that exist in the side module + if ('__start_em_asm' in moduleExports) { + var start = moduleExports['__start_em_asm']; + var stop = moduleExports['__stop_em_asm']; + {{{ from64('start') }}} + {{{ from64('stop') }}} + while (start < stop) { + var jsString = UTF8ToString(start); + addEmAsm(start, jsString); + start = HEAPU8.indexOf(0, start) + 1; + } + } + + function addEmJs(name, cSig, body) { + // The signature here is a C signature (e.g. "(int foo, char* bar)"). + // See `create_em_js` in emcc.py` for the build-time version of this + // code. + var jsArgs = []; + cSig = cSig.slice(1, -1) + if (cSig != 'void') { + cSig = cSig.split(','); + for (var i in cSig) { + var jsArg = cSig[i].split(' ').pop(); + jsArgs.push(jsArg.replaceAll('*', '')); + } + } + var func = `(${jsArgs}) => ${body};`; +#if DYLINK_DEBUG + dbg(`adding new EM_JS function: ${jsArgs} = ${func}`); +#endif + {{{ makeEval('moduleExports[name] = eval(func)') }}}; + } + + for (var name in moduleExports) { + if (name.startsWith('__em_js__')) { + var start = moduleExports[name] + var jsString = UTF8ToString({{{ from64Expr('start') }}}); + // EM_JS strings are stored in the data section in the form + // SIG<::>BODY. + var parts = jsString.split('<::>'); + addEmJs(name.replace('__em_js__', ''), parts[0], parts[1]); + delete moduleExports[name]; + } + } +#endif + + // initialize the module +#if PTHREADS + // Only one thread should call __wasm_call_ctors, but all threads need + // to call _emscripten_tls_init + registerTLSInit(moduleExports['_emscripten_tls_init'], instance.exports, metadata) + if (firstLoad) { +#endif + var applyRelocs = moduleExports['__wasm_apply_data_relocs']; + if (applyRelocs) { + if (runtimeInitialized) { +#if DYLINK_DEBUG + dbg('applyRelocs'); +#endif + applyRelocs(); + } else { + __RELOC_FUNCS__.push(applyRelocs); + } + } + var init = moduleExports['__wasm_call_ctors']; + if (init) { + if (runtimeInitialized) { + init(); + } else { + // we aren't ready to run compiled code yet + __ATINIT__.push(init); + } + } +#if PTHREADS + } +#endif + return moduleExports; + } + + if (flags.loadAsync) { + if (binary instanceof WebAssembly.Module) { + var instance = new WebAssembly.Instance(binary, info); + return Promise.resolve(postInstantiation(binary, instance)); + } + return WebAssembly.instantiate(binary, info).then( + (result) => postInstantiation(result.module, result.instance) + ); + } + + var module = binary instanceof WebAssembly.Module ? binary : new WebAssembly.Module(binary); + var instance = new WebAssembly.Instance(module, info); + return postInstantiation(module, instance); + } + + // now load needed libraries and the module itself. + if (flags.loadAsync) { + return metadata.neededDynlibs + .reduce((chain, dynNeeded) => chain.then(() => + loadDynamicLibrary(dynNeeded, flags, localScope) + ), Promise.resolve()) + .then(loadModule); + } + + metadata.neededDynlibs.forEach((needed) => loadDynamicLibrary(needed, flags, localScope)); + return loadModule(); + }, + +#if STACK_OVERFLOW_CHECK >= 2 + // Sometimes we load libraries before runtime initialization. In this case + // we delay calling __set_stack_limits (which must be called for each + // module). + $setDylinkStackLimits: (stackTop, stackMax) => { + for (var name in LDSO.loadedLibsByName) { +#if DYLINK_DEBUG + dbg(`setDylinkStackLimits for '${name}'`); +#endif + var lib = LDSO.loadedLibsByName[name]; + lib.exports['__set_stack_limits']?.({{{ to64("stackTop") }}}, {{{ to64("stackMax") }}}); + } + }, +#endif + + $newDSO: (name, handle, syms) => { + var dso = { + refcount: Infinity, + name, + exports: syms, + global: true, + }; + LDSO.loadedLibsByName[name] = dso; + if (handle != undefined) { + LDSO.loadedLibsByHandle[handle] = dso; + } + return dso; + }, + + // loadDynamicLibrary loads dynamic library @ lib URL / path and returns + // handle for loaded DSO. + // + // Several flags affect the loading: + // + // - if flags.global=true, symbols from the loaded library are merged into global + // process namespace. Flags.global is thus similar to RTLD_GLOBAL in ELF. + // + // - if flags.nodelete=true, the library will be never unloaded. Flags.nodelete + // is thus similar to RTLD_NODELETE in ELF. + // + // - if flags.loadAsync=true, the loading is performed asynchronously and + // loadDynamicLibrary returns corresponding promise. + // + // If a library was already loaded, it is not loaded a second time. However + // flags.global and flags.nodelete are handled every time a load request is made. + // Once a library becomes "global" or "nodelete", it cannot be removed or unloaded. + $loadDynamicLibrary__deps: ['$LDSO', '$loadWebAssemblyModule', + '$isInternalSym', '$mergeLibSymbols', '$newDSO', + '$asyncLoad', +#if FILESYSTEM + '$preloadedWasm', +#endif +#if DYNCALLS || !WASM_BIGINT + '$registerDynCallSymbols', +#endif + ], + $loadDynamicLibrary__docs: ` + /** + * @param {number=} handle + * @param {Object=} localScope + */`, + $loadDynamicLibrary: function(libName, flags = {global: true, nodelete: true}, localScope, handle) { +#if DYLINK_DEBUG + dbg(`loadDynamicLibrary: ${libName} handle: ${handle}`); + dbg(`existing: ${Object.keys(LDSO.loadedLibsByName)}`); +#endif + // when loadDynamicLibrary did not have flags, libraries were loaded + // globally & permanently + + var dso = LDSO.loadedLibsByName[libName]; + if (dso) { + // the library is being loaded or has been loaded already. +#if ASSERTIONS + assert(dso.exports !== 'loading', `Attempt to load '${libName}' twice before the first load completed`); +#endif + if (!flags.global) { + if (localScope) { + Object.assign(localScope, dso.exports); + } +#if DYNCALLS || !WASM_BIGINT + registerDynCallSymbols(dso.exports); +#endif + } else if (!dso.global) { + // The library was previously loaded only locally but not + // we have a request with global=true. + dso.global = true; + mergeLibSymbols(dso.exports, libName) + } + // same for "nodelete" + if (flags.nodelete && dso.refcount !== Infinity) { + dso.refcount = Infinity; + } + dso.refcount++ + if (handle) { + LDSO.loadedLibsByHandle[handle] = dso; + } + return flags.loadAsync ? Promise.resolve(true) : true; + } + + // allocate new DSO + dso = newDSO(libName, handle, 'loading'); + dso.refcount = flags.nodelete ? Infinity : 1; + dso.global = flags.global; + + // libName -> libData + function loadLibData() { +#if PTHREADS + var sharedMod = sharedModules[libName]; +#if DYLINK_DEBUG + dbg(`checking sharedModules: ${libName}: ${sharedMod ? 'found' : 'not found'}`); +#endif + if (sharedMod) { + return flags.loadAsync ? Promise.resolve(sharedMod) : sharedMod; + } +#endif + + // for wasm, we can use fetch for async, but for fs mode we can only imitate it + if (handle) { + var data = {{{ makeGetValue('handle', C_STRUCTS.dso.file_data, '*') }}}; + var dataSize = {{{ makeGetValue('handle', C_STRUCTS.dso.file_data_size, '*') }}}; + if (data && dataSize) { + var libData = HEAP8.slice(data, data + dataSize); + return flags.loadAsync ? Promise.resolve(libData) : libData; + } + } + + var libFile = locateFile(libName); + if (flags.loadAsync) { + return asyncLoad(libFile); + } + + // load the binary synchronously + if (!readBinary) { + throw new Error(`${libFile}: file not found, and synchronous loading of external files is not available`); + } + return readBinary(libFile); + } + + // libName -> exports + function getExports() { +#if FILESYSTEM + // lookup preloaded cache first + var preloaded = preloadedWasm[libName]; +#if DYLINK_DEBUG + dbg(`checking preloadedWasm: ${libName}: ${preloaded ? 'found' : 'not found'}`); +#endif + if (preloaded) { + return flags.loadAsync ? Promise.resolve(preloaded) : preloaded; + } +#endif + + // module not preloaded - load lib data and create new module from it + if (flags.loadAsync) { + return loadLibData().then((libData) => loadWebAssemblyModule(libData, flags, libName, localScope, handle)); + } + + return loadWebAssemblyModule(loadLibData(), flags, libName, localScope, handle); + } + + // module for lib is loaded - update the dso & global namespace + function moduleLoaded(exports) { + if (dso.global) { + mergeLibSymbols(exports, libName); + } else if (localScope) { + Object.assign(localScope, exports); +#if DYNCALLS || !WASM_BIGINT + registerDynCallSymbols(exports); +#endif + } + dso.exports = exports; + } + + if (flags.loadAsync) { +#if DYLINK_DEBUG + dbg("loadDynamicLibrary: done (async)"); +#endif + return getExports().then((exports) => { + moduleLoaded(exports); + return true; + }); + } + + moduleLoaded(getExports()); +#if DYLINK_DEBUG + dbg("loadDynamicLibrary: done"); +#endif + return true; + }, + + $loadDylibs__internal: true, + $loadDylibs__deps: ['$loadDynamicLibrary', '$reportUndefinedSymbols'], + $loadDylibs: () => { + if (!dynamicLibraries.length) { +#if DYLINK_DEBUG + dbg('loadDylibs: no libraries to preload'); +#endif + reportUndefinedSymbols(); + return; + } + +#if DYLINK_DEBUG + dbg(`loadDylibs: ${dynamicLibraries}`); +#endif + + // Load binaries asynchronously + addRunDependency('loadDylibs'); + dynamicLibraries + .reduce((chain, lib) => chain.then(() => + loadDynamicLibrary(lib, {loadAsync: true, global: true, nodelete: true, allowUndefined: true}) + ), Promise.resolve()) + .then(() => { + // we got them all, wonderful + reportUndefinedSymbols(); + removeRunDependency('loadDylibs'); + #if DYLINK_DEBUG + dbg('loadDylibs done!'); + #endif + }); + }, + + // void* dlopen(const char* filename, int flags); + $dlopenInternal__deps: ['$ENV', '$dlSetError', '$PATH'], + $dlopenInternal: (handle, jsflags) => { + // void *dlopen(const char *file, int mode); + // http://pubs.opengroup.org/onlinepubs/009695399/functions/dlopen.html + var filename = UTF8ToString(handle + {{{ C_STRUCTS.dso.name }}}); + var flags = {{{ makeGetValue('handle', C_STRUCTS.dso.flags, 'i32') }}}; +#if DYLINK_DEBUG + dbg(`dlopenInternal: ${filename}`); +#endif + filename = PATH.normalize(filename); + var searchpaths = []; + + var global = Boolean(flags & {{{ cDefs.RTLD_GLOBAL }}}); + var localScope = global ? null : {}; + + // We don't care about RTLD_NOW and RTLD_LAZY. + var combinedFlags = { + global, + nodelete: Boolean(flags & {{{ cDefs.RTLD_NODELETE }}}), + loadAsync: jsflags.loadAsync, + } + + if (jsflags.loadAsync) { + return loadDynamicLibrary(filename, combinedFlags, localScope, handle); + } + + try { + return loadDynamicLibrary(filename, combinedFlags, localScope, handle) + } catch (e) { +#if ASSERTIONS + err(`Error in loading dynamic library ${filename}: ${e}`); +#endif + dlSetError(`Could not load dynamic lib: ${filename}\n${e}`); + return 0; + } + }, + + _dlopen_js__deps: ['$dlopenInternal'], +#if ASYNCIFY + _dlopen_js__async: true, +#endif + _dlopen_js: (handle) => { +#if ASYNCIFY + return Asyncify.handleSleep((wakeUp) => { + dlopenInternal(handle, { loadAsync: true }) + .then(wakeUp) + .catch(() => wakeUp(0)); + }); +#else + return dlopenInternal(handle, { loadAsync: false }); +#endif + }, + + // Async version of dlopen. + _emscripten_dlopen_js__deps: ['$dlopenInternal', '$callUserCallback', '$dlSetError'], + _emscripten_dlopen_js: (handle, onsuccess, onerror, user_data) => { + /** @param {Object=} e */ + function errorCallback(e) { + var filename = UTF8ToString(handle + {{{ C_STRUCTS.dso.name }}}); + dlSetError(`'Could not load dynamic lib: ${filename}\n${e}`); + {{{ runtimeKeepalivePop() }}} + callUserCallback(() => {{{ makeDynCall('vpp', 'onerror') }}}(handle, user_data)); + } + function successCallback() { + {{{ runtimeKeepalivePop() }}} + callUserCallback(() => {{{ makeDynCall('vpp', 'onsuccess') }}}(handle, user_data)); + } + + {{{ runtimeKeepalivePush() }}} + var promise = dlopenInternal(handle, { loadAsync: true }); + if (promise) { + promise.then(successCallback, errorCallback); + } else { + errorCallback(); + } + }, + + _dlsym_catchup_js: (handle, symbolIndex) => { +#if DYLINK_DEBUG + dbg("_dlsym_catchup: handle=" + ptrToString(handle) + " symbolIndex=" + symbolIndex); +#endif + var lib = LDSO.loadedLibsByHandle[handle]; + var symDict = lib.exports; + var symName = Object.keys(symDict)[symbolIndex]; + var sym = symDict[symName]; + var result = addFunction(sym, sym.sig); +#if DYLINK_DEBUG + dbg(`_dlsym_catchup: result=${result}`); +#endif + return result; + }, + + // void* dlsym(void* handle, const char* symbol); + _dlsym_js__deps: ['$dlSetError', '$getFunctionAddress', '$addFunction'], + _dlsym_js: (handle, symbol, symbolIndex) => { + // void *dlsym(void *restrict handle, const char *restrict name); + // http://pubs.opengroup.org/onlinepubs/009695399/functions/dlsym.html + symbol = UTF8ToString(symbol); +#if DYLINK_DEBUG + dbg(`dlsym_js: ${symbol}`); +#endif + var result; + var newSymIndex; + + var lib = LDSO.loadedLibsByHandle[handle]; +#if ASSERTIONS + assert(lib, `Tried to dlsym() from an unopened handle: ${handle}`); +#endif + if (!lib.exports.hasOwnProperty(symbol) || lib.exports[symbol].stub) { + dlSetError(`Tried to lookup unknown symbol "${symbol}" in dynamic lib: ${lib.name}`) + return 0; + } + newSymIndex = Object.keys(lib.exports).indexOf(symbol); +#if !WASM_BIGINT + var origSym = 'orig$' + symbol; + result = lib.exports[origSym]; + if (result) { + newSymIndex = Object.keys(lib.exports).indexOf(origSym); + } + else +#endif + result = lib.exports[symbol]; + + if (typeof result == 'function') { +#if DYLINK_DEBUG + dbg(`dlsym_js: ${symbol} getting table slot for: ${result}`); +#endif + +#if ASYNCIFY + // Asyncify wraps exports, and we need to look through those wrappers. + if ('orig' in result) { + result = result.orig; + } +#endif + var addr = getFunctionAddress(result); + if (addr) { +#if DYLINK_DEBUG + dbg(`symbol already exists in table: ${symbol}`); +#endif + result = addr; + } else { + // Insert the function into the wasm table. If its a direct wasm + // function the second argument will not be needed. If its a JS + // function we rely on the `sig` attribute being set based on the + // `__sig` specified in library JS file. + result = addFunction(result, result.sig); +#if DYLINK_DEBUG + dbg(`adding symbol to table: ${symbol}`); +#endif + {{{ makeSetValue('symbolIndex', 0, 'newSymIndex', '*') }}}; + } + } +#if DYLINK_DEBUG + dbg(`dlsym_js: ${symbol} -> ${result}`); +#endif + return result; + }, +}; + +addToLibrary(LibraryDylink); From d7577b660b22091d2abc52e293bfb33667054b1f Mon Sep 17 00:00:00 2001 From: pmp-p Date: Thu, 31 Jul 2025 13:50:41 +0200 Subject: [PATCH 04/11] compare hotfixed --- .buildconfig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildconfig b/.buildconfig index a518dff8eedcc..ac9d2913c9535 100644 --- a/.buildconfig +++ b/.buildconfig @@ -1,6 +1,6 @@ PG_VERSION=17.5 PG_BRANCH=REL_17_5_WASM -SDK_VERSION=3.1.74.12.0 +SDK_VERSION=3.1.74.11.11 SDKROOT=/tmp/sdk GETZIC=false ZIC=/usr/sbin/zic From 7dbc9f69729a5a50effc0851a3c5d7ac72411c60 Mon Sep 17 00:00:00 2001 From: Marcus Pousette Date: Mon, 8 Sep 2025 13:54:09 +0200 Subject: [PATCH 05/11] Improve MemoryContext management to prevent memory exhaustion between queries (#47) * Improve MemoryContext management to prevent memory exhaustion between queries * Use MemoryContextReset; drop legacy MemoryContextResetAndDeleteChildren macro * Cleanup: remove extraneous comment; keep MemoryContextReset only --- pglite-wasm/interactive_one.c | 2 +- pglite-wasm/pg_main.c | 2 -- pglite-wasm/pgl_mains.c | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/pglite-wasm/interactive_one.c b/pglite-wasm/interactive_one.c index fc3590961d5d6..124e99c5059f4 100644 --- a/pglite-wasm/interactive_one.c +++ b/pglite-wasm/interactive_one.c @@ -416,7 +416,7 @@ if (cma_rsize<0) } MemoryContextSwitchTo(MessageContext); - MemoryContextResetAndDeleteChildren(MessageContext); + MemoryContextReset(MessageContext); initStringInfo(&input_message); diff --git a/pglite-wasm/pg_main.c b/pglite-wasm/pg_main.c index 65e5d7ac01d2c..07c21cf1b5b24 100644 --- a/pglite-wasm/pg_main.c +++ b/pglite-wasm/pg_main.c @@ -28,8 +28,6 @@ // globals -#define MemoryContextResetAndDeleteChildren(...) -// #define SpinLockInit(...) diff --git a/pglite-wasm/pgl_mains.c b/pglite-wasm/pgl_mains.c index 98f425a6e4116..7dc200745a4d7 100644 --- a/pglite-wasm/pgl_mains.c +++ b/pglite-wasm/pgl_mains.c @@ -45,7 +45,7 @@ interactive_file() { * query input buffer in the cleared MessageContext. */ MemoryContextSwitchTo(MessageContext); - MemoryContextResetAndDeleteChildren(MessageContext); + MemoryContextReset(MessageContext); initStringInfo(&input_message); inBuf = &input_message; From c0269765ed89b8df428a91192d16ffbfdb8176e0 Mon Sep 17 00:00:00 2001 From: Tudor Zaharia <37596026+tdrz@users.noreply.github.com> Date: Wed, 17 Sep 2025 15:46:51 +0200 Subject: [PATCH 06/11] Switch to vanilla emsdk for building pglite (#48) * first pass * skip postgresql tests as usual * preloaded files; pglite.data file autotools handling * new pglite and install-pglite Makefile targets; workflow succesful; * add pglite.{data,js,html} to gitignore * remove the need for emsdk-shared script * add password and PGPASSFILE static files * use system zic from /usr/sbin/zic for timezone * use static files from repo when preloading password and PGPASSFILE * add empty file * builder dockerfile; external deps source code * use -sMAIN_MODULE=2 which reduces size code * building with -sMAIN_MODULE=2 to reduce size * use file containing needed EXPORTED_FUNCTIONS - tested on data generated from the *.o files of plpgsql + the already predefined ones; -sDYLINK_DEBUG=2 flag is very useful in debugging emscripten's dynamic linking; generating the EXPORTED_FUNCTIONS needs to be automated * automate creation of imports needed by plpgsql; automate creation of EXPORTED_FUNCTIONS emscripten parameter; * infrastructure for contrib extensions * more excluded imports * link pglite.wasm against more libs so the exports are visible for emscripten * more EXPORTED_FUNCTIONS excludes * more EXPORTED_FUNCTIONS includes * extended docker image to contain the exports of the prebuilt libraries * change the exports format of shared libs * extend backend makefile to better handle EXPORTED_FUNCTIONS depending on shared libs' exports * git: ignore .map files * added pg_ivm as submodule; added Makefile to build such submodules * added pgvector as submodule and updated the Makefile to build it * use env var to point to llvm-nm for exports and imports defs * exclude libossp-uuid exports and dont link libossp-uuid to the exe * git ignore excluded.imports file * new simple bash script to build everything; allow debug builds * cleanup * added bashs script to clean everything * unset DEBUG var if it is false - there is a bug somewhere and until we fix that, this works; bug fix in passing PGLITE_CFLAGS * further simplify build-with-docker2.sh script * build pglite script remove SOURCE_MAP_PREFIXES * comments * rename script * add 2 more exported functions; sort the always included imports * separate dockerfiles for different archs - this is needed because emsdk does not publish different archs under the same tag; hopefully this will be fixed in the future * contrib extensions packaging; integration into the build of the said extensions * better error message when building/disting contribs * change custom build root for pglite contrib * add LDFLAGS_SL="-sSIDE_MODULE=1" when building contrib extensions * renamed pgvector submodule to vector * dist extra extensions insted of install (packaging pgvector and pg_ivm into tar.gz) * use pushed docker image * build only using native build platform, since wasm is the same on all platforms * remove unused dockerfiles * cleanup src/backend/Makefile * remove wasm-builder folder * added comment to other extensions Makefile * comments * remove binaries used for pglite-builder * remove config.site since the param that it contains is already passed in the command line * update clean-pglite.sh script --------- Co-authored-by: tudor --- .gitignore | 9 + .gitmodules | 6 + build-pglite.sh | 46 +++++ build-with-docker.sh | 13 ++ clean-pglite.sh | 6 + contrib/Makefile | 1 + contrib/dist.mk | 27 +++ other/PGPASSFILE | 3 + other/empty | 0 other/password | 1 + pglite-wasm/builder/Dockerfile | 89 +++++++++ pglite-wasm/excluded.pglite.imports | 32 ++++ pglite-wasm/included.pglite.imports | 38 ++++ pglite-wasm/pg_main.c | 7 +- pglite/Makefile | 35 ++++ pglite/pg_ivm | 1 + pglite/vector | 1 + src/Makefile.shlib | 5 +- src/backend/Makefile | 79 +++++++- src/backend/libpq/pqcomm.c | 4 + src/bin/initdb/Makefile | 9 + src/bin/pg_amcheck/Makefile | 9 + src/bin/pg_archivecleanup/Makefile | 9 + src/bin/pg_basebackup/Makefile | 15 ++ src/bin/pg_checksums/Makefile | 9 + src/bin/pg_combinebackup/Makefile | 9 + src/bin/pg_config/Makefile | 27 +++ src/bin/pg_controldata/Makefile | 9 + src/bin/pg_ctl/Makefile | 9 + src/bin/pg_dump/Makefile | 11 ++ src/bin/pg_resetwal/Makefile | 9 + src/bin/pg_rewind/Makefile | 9 + src/bin/pg_test_fsync/Makefile | 9 + src/bin/pg_test_timing/Makefile | 9 + src/bin/pg_upgrade/Makefile | 9 + src/bin/pg_verifybackup/Makefile | 9 + src/bin/pg_waldump/Makefile | 9 + src/bin/pg_walsummary/Makefile | 9 + src/bin/pgbench/Makefile | 9 + src/bin/psql/Makefile | 9 + src/bin/scripts/Makefile | 16 ++ src/include/port/emscripten.h | 7 +- src/include/port/pg_debug.h | 8 + src/include/port/sdk_port.h | 262 +++++++++++++++++++++++++++ src/include/port/wasm_common.h | 2 +- src/interfaces/ecpg/preproc/Makefile | 9 + src/makefiles/Makefile.emscripten | 6 + src/makefiles/pgxs.mk | 22 +++ src/pl/plpgsql/src/Makefile | 12 ++ src/test/Makefile | 8 + src/test/isolation/Makefile | 8 + src/timezone/Makefile | 7 + wasm-build/{ => include}/sdk_port.h | 0 wasm-builder/Dockerfile | 23 --- wasm-builder/build-with-docker.sh | 69 ------- 55 files changed, 968 insertions(+), 100 deletions(-) create mode 100644 .gitmodules create mode 100755 build-pglite.sh create mode 100755 build-with-docker.sh create mode 100755 clean-pglite.sh create mode 100644 contrib/dist.mk create mode 100644 other/PGPASSFILE create mode 100644 other/empty create mode 100644 other/password create mode 100644 pglite-wasm/builder/Dockerfile create mode 100644 pglite-wasm/excluded.pglite.imports create mode 100644 pglite-wasm/included.pglite.imports create mode 100644 pglite/Makefile create mode 160000 pglite/pg_ivm create mode 160000 pglite/vector create mode 100644 src/include/port/pg_debug.h create mode 100644 src/include/port/sdk_port.h rename wasm-build/{ => include}/sdk_port.h (100%) delete mode 100644 wasm-builder/Dockerfile delete mode 100755 wasm-builder/build-with-docker.sh diff --git a/.gitignore b/.gitignore index e76ef7cc785ea..2e67d7c8e6671 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,12 @@ lib*.pc /tmp_install/ /portlock/ /dist + +# emscripten build excludes +*.cjs +*.wasm +pglite.data +pglite.js +pglite.html +*.map +pglite-wasm/excluded.imports diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000000000..9317a32956edc --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "pglite/pg_ivm"] + path = pglite/pg_ivm + url = https://github.com/sraoss/pg_ivm.git +[submodule "pglite/pgvector"] + path = pglite/vector + url = https://github.com/pgvector/pgvector.git diff --git a/build-pglite.sh b/build-pglite.sh new file mode 100755 index 0000000000000..7fa05c3900d95 --- /dev/null +++ b/build-pglite.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +### NOTES ### +# $INSTALL_PREFIX is expected to point to the installation folder of various libraries built to wasm (see pglite-builder) +############# + +# final output folder +INSTALL_FOLDER=${INSTALL_FOLDER:-"/install/pglite"} + +# build with optimizations by default aka release +PGLITE_CFLAGS="-O2" +if [ "$DEBUG" = true ] +then + echo "pglite: building debug version." + PGLITE_CFLAGS="-g -gsource-map --no-wasm-opt" +else + echo "pglite: building release version." + # we shouldn't need to do this, but there's a bug somewhere that prevents a successful build if this is set + unset DEBUG +fi + +echo "pglite: PGLITE_CFLAGS=$PGLITE_CFLAGS" + +# Step 1: configure the project +LDFLAGS="-sWASM_BIGINT -sUSE_PTHREADS=0" CFLAGS="${PGLITE_CFLAGS} -sWASM_BIGINT -fpic -sENVIRONMENT=node,web,worker -sSUPPORT_LONGJMP=emscripten -DPYDK=1 -DCMA_MB=12 -Wno-declaration-after-statement -Wno-macro-redefined -Wno-unused-function -Wno-missing-prototypes -Wno-incompatible-pointer-types" emconfigure ./configure ac_cv_exeext=.cjs --disable-spinlocks --disable-largefile --without-llvm --without-pam --disable-largefile --with-openssl=no --without-readline --without-icu --with-includes=$INSTALL_PREFIX/include:$INSTALL_PREFIX/include/libxml2 --with-libraries=$INSTALL_PREFIX/lib --with-uuid=ossp --with-zlib --with-libxml --with-libxslt --with-template=emscripten --prefix=$INSTALL_FOLDER || { echo 'error: emconfigure failed' ; exit 11; } + +# Step 2: make and install all except pglite +emmake make PORTNAME=emscripten -j || { echo 'error: emmake make PORTNAME=emscripten -j' ; exit 21; } +emmake make PORTNAME=emscripten install || { echo 'error: emmake make PORTNAME=emscripten install' ; exit 22; } + +# Step 3.1: make all contrib extensions - do not install +emmake make PORTNAME=emscripten LDFLAGS_SL="-sSIDE_MODULE=1" -C contrib/ -j || { echo 'error: emmake make PORTNAME=emscripten -C contrib/ -j' ; exit 31; } +# Step 3.2: make dist contrib extensions - this will create an archive for each extension +emmake make PORTNAME=emscripten -C contrib/ dist || { echo 'error: emmake make PORTNAME=emscripten -C contrib/ dist' ; exit 32; } +# the above will also create a file with the imports that each extension needs - we pass these as input in the next step for emscripten to keep alive + +# Step 4: make and dist other extensions +SAVE_PATH=$PATH +PATH=$PATH:$INSTALL_FOLDER/bin +emmake make OPTFLAGS="" PORTNAME=emscripten -j -C pglite || { echo 'error: emmake make OPTFLAGS="" PORTNAME=emscripten -j -C pglite' ; exit 41; } +emmake make OPTFLAGS="" PORTNAME=emscripten LDFLAGS_SL="-sSIDE_MODULE=1" -C pglite/ dist || { echo 'error: make OPTFLAGS="" PORTNAME=emscripten LDFLAGS_SL="-sSIDE_MODULE=1" -C pglite/ dist ' ; exit 42; } +PATH=$SAVE_PATH + +# Step 5: make and install pglite +# Building pglite itself needs to be the last step because of the PRELOAD_FILES parameter (a list of files and folders) need to be available. +PGLITE_CFLAGS=$PGLITE_CFLAGS emmake make PORTNAME=emscripten -j -C src/backend/ install-pglite || { echo 'emmake make OPTFLAGS="" PORTNAME=emscripten -j -C pglite' ; exit 51; } diff --git a/build-with-docker.sh b/build-with-docker.sh new file mode 100755 index 0000000000000..d3d31a17fdc75 --- /dev/null +++ b/build-with-docker.sh @@ -0,0 +1,13 @@ +# although we could use any path inside docker, using the same path as on the host +# allows the DWARF info (when building in DEBUG) to contain the correct file paths +DOCKER_WORKSPACE=$(pwd) + +docker run $@ \ + --rm \ + -e DEBUG=${DEBUG:-false} \ + --workdir=${DOCKER_WORKSPACE} \ + -v .:${DOCKER_WORKSPACE}:rw \ + -v ./dist:/install/pglite:rw \ + electricsql/pglite-builder:3.1.74_1 \ + ./build-pglite.sh + diff --git a/clean-pglite.sh b/clean-pglite.sh new file mode 100755 index 0000000000000..3d6c86458baa6 --- /dev/null +++ b/clean-pglite.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +emmake make -C src/backend uninstall; emmake make -C src/backend clean; +emmake make -C pglite/ clean; emmake make -C pglite/ uninstall; +emmake make -C contrib/ clean; emmake make -C contrib/ uninstall; emmake make -C pglite clean; emmake make -C pglite uninstall; +emmake make clean; emmake make uninstal \ No newline at end of file diff --git a/contrib/Makefile b/contrib/Makefile index abd780f277405..603636cd4d248 100644 --- a/contrib/Makefile +++ b/contrib/Makefile @@ -90,6 +90,7 @@ endif # Missing: # start-scripts \ (does not have a makefile) +include ./dist.mk $(recurse) $(recurse_always) diff --git a/contrib/dist.mk b/contrib/dist.mk new file mode 100644 index 0000000000000..df3933f01fb82 --- /dev/null +++ b/contrib/dist.mk @@ -0,0 +1,27 @@ +# contrib/dist.mk +# +# Package each contrib extension into its own .tar.gz archive + +prefix ?= /install/pglite +CONTRIB_BUILD_ROOT := /tmp/extensions/build +ARCHIVE_DIR := /install/pglite/extensions + +CONTRIBS := $(SUBDIRS) + +# Default target: build tarballs for all contribs +dist: $(addsuffix .tar.gz,$(CONTRIBS)) + +# Pattern rule: build $(EXT).tar.gz for each contrib +%.tar.gz: + @echo "=== Staging $* ===" + rm -rf $(CONTRIB_BUILD_ROOT)/$* + bash -c 'mkdir -p $(CONTRIB_BUILD_ROOT)/$*/$(prefix)/{bin,lib,share/extension,share/doc,share/postgresql/extension,share/postgresql/tsearch_data,include}' + $(MAKE) -C $* install DESTDIR=$(CONTRIB_BUILD_ROOT)/$* + @echo "=== Packaging $* ===" + mkdir -p $(ARCHIVE_DIR) + cd $(CONTRIB_BUILD_ROOT)/$*/$(prefix) && \ + files=$$(find . -type f -o -type l | sed 's|^\./||') && \ + tar -czf $(ARCHIVE_DIR)/$*.tar.gz $$files +# tar -C $(CONTRIB_BUILD_ROOT)/$*/$(prefix) -czf $(ARCHIVE_DIR)/$*.tar.gz . + +.PHONY: dist diff --git a/other/PGPASSFILE b/other/PGPASSFILE new file mode 100644 index 0000000000000..b8a0b5b6a58ac --- /dev/null +++ b/other/PGPASSFILE @@ -0,0 +1,3 @@ +localhost:5432:postgres:password:md532e12f215ba27cb750c9e093ce4b5127 +localhost:5432:postgres:postgres:md53175bce1d3201d16594cebf9d7eb3f9d +localhost:5432:postgres:login:md5d5745f9425eceb269f9fe01d0bef06ff diff --git a/other/empty b/other/empty new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/other/password b/other/password new file mode 100644 index 0000000000000..f3097ab13082b --- /dev/null +++ b/other/password @@ -0,0 +1 @@ +password diff --git a/pglite-wasm/builder/Dockerfile b/pglite-wasm/builder/Dockerfile new file mode 100644 index 0000000000000..d8f5d931f75b1 --- /dev/null +++ b/pglite-wasm/builder/Dockerfile @@ -0,0 +1,89 @@ +ARG EMSDK_VER=3.1.74 +ARG BUILDPLATFORM +# we only need to build on one platform, since we're interested in the WASM output +# building on the native (BUILDPLATFORM) is much faster, so use that +# remove "-arm64" suffix if building on x86_64 +FROM --platform=$BUILDPLATFORM emscripten/emsdk:${EMSDK_VER}-arm64 AS builder + +ENV LLVM_NM=/emsdk/upstream/bin/llvm-nm + +RUN apt update && apt upgrade -y && apt install -y \ + xz-utils autoconf libtool automake pkgconf bison flex + +SHELL ["/bin/bash", "-c"] + +RUN mkdir -p /install/libs /install/exports + +WORKDIR /install/libs + +# zlib CAN be installed by using the emsdk like this +# RUN embuilder --pic --verbose build zlib + +WORKDIR /src + +# ENV EMCC_COMMON_FLAGS="-fPIC -sWASM_BIGINT -sMIN_SAFARI_VERSION=150000 -D__PYDK__=1 -O2 -m32 -D_FILE_OFFSET_BITS=64 -sSUPPORT_LONGJMP=emscripten -mno-bulk-memory -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-extended-const -mno-atomics -mno-tail-call -mno-multivalue -mno-relaxed-simd -mno-simd128 -mno-multimemory -mno-exception-handling -Wno-unused-command-line-argument -Wno-unreachable-code-fallthrough -Wno-unused-function -Wno-invalid-noreturn -Wno-declaration-after-statement -Wno-invalid-noreturn" +ENV EMCC_COMMON_FLAGS="-O2 -fPIC" + +WORKDIR /src +RUN curl -L https://www.zlib.net/zlib-1.3.1.tar.gz | tar -xz +WORKDIR /src/zlib-1.3.1 +RUN CFLAGS="${EMCC_COMMON_FLAGS}" CXXFLAGS="${EMCC_COMMON_FLAGS}" emconfigure ./configure --static --prefix=/install/libs +RUN emmake make -j && emmake make install +RUN ${LLVM_NM} /install/libs/lib/libz.a | awk '$2 ~ /^[TDB]$/ {print $3}' | sed '/^$/d' | sort -u > /install/exports/libz.exports + +WORKDIR /src +RUN curl -L https://gitlab.gnome.org/GNOME/libxml2/-/archive/v2.14.5/libxml2-v2.14.5.tar.gz | tar -xz +WORKDIR /src/libxml2-v2.14.5 +RUN ./autogen.sh --with-python=no +RUN CFLAGS="${EMCC_COMMON_FLAGS}" CXXFLAGS="${EMCC_COMMON_FLAGS}" emconfigure ./configure --enable-shared=no --enable-static=yes --with-python=no --prefix=/install/libs +RUN emmake make -j && emmake make install +# extract exported symbols - useful for passing them to emscripten as EXPORTED_FUNCTIONS +RUN ${LLVM_NM} /install/libs/lib/libxml2.a | awk '$2 ~ /^[TDB]$/ {print $3}' | sed '/^$/d' | sort -u > /install/exports/libxml2.exports + +WORKDIR /src +RUN curl -L https://gitlab.gnome.org/GNOME/libxslt/-/archive/v1.1.43/libxslt-v1.1.43.tar.gz | tar -xz +WORKDIR /src/libxslt-v1.1.43 +RUN ./autogen.sh --with-python=no +RUN CFLAGS="${EMCC_COMMON_FLAGS}" CXXFLAGS="${EMCC_COMMON_FLAGS}" emconfigure ./configure --enable-shared=no --enable-static=yes --with-python=no --prefix=/install/libs --with-libxml-src=/src/libxml2-v2.14.5/ --with-pic=yes +RUN emmake make -j && emmake make install +# extract exported symbols - useful for passing them to emscripten as EXPORTED_FUNCTIONS +RUN ${LLVM_NM} /install/libs/lib/libxslt.a | awk '$2 ~ /^[TDB]$/ {print $3}' | sed '/^$/d' | sort -u > /install/exports/libxslt.exports + +WORKDIR /src +RUN curl -L https://github.com/openssl/openssl/releases/download/openssl-3.0.17/openssl-3.0.17.tar.gz | tar xz +WORKDIR /src/openssl-3.0.17 +RUN emconfigure ./Configure no-tests linux-generic64 --prefix=/install/libs +RUN sed -i 's|^CROSS_COMPILE.*$|CROSS_COMPILE=|g' Makefile # see https://github.com/emscripten-core/emscripten/issues/19597#issue-1754476454 +RUN emmake make -j && emmake make install +# extract exported symbols - useful for passing them to emscripten as EXPORTED_FUNCTIONS +RUN ${LLVM_NM} /install/libs/lib/libssl.a | awk '$2 ~ /^[TDB]$/ {print $3}' | sed '/^$/d' | sort -u > /install/exports/libssl.exports + +WORKDIR /src +RUN curl -L ftp://ftp.ossp.org/pkg/lib/uuid/uuid-1.6.2.tar.gz | tar xz +WORKDIR /src/uuid-1.6.2 +RUN CFLAGS="${EMCC_COMMON_FLAGS}" CXXFLAGS="${EMCC_COMMON_FLAGS}" emconfigure ./configure --build=aarch64-unknown-linux-gnu --enable-shared=no --enable-static=yes --with-perl=no --with-perl-compat=no --prefix=/install/libs --with-php=no --with-pic=yes +RUN emmake make -j && emmake make install || true # install tries to strip the wasm, but it doesnt recognize the format, so ignore atm +WORKDIR /install/libs/lib +RUN ln -s libuuid.a libossp-uuid.a # contrib extensions use -lossp-uuid +RUN ${LLVM_NM} /install/libs/lib/libossp-uuid.a | awk '$2 ~ /^[TDB]$/ {print $3}' | sed '/^$/d' | sort -u > /install/exports/libossp-uuid.exports + +ARG TARGETARCH +FROM emscripten/emsdk:${EMSDK_VER} AS base-amd64 +FROM emscripten/emsdk:${EMSDK_VER}-arm64 AS base-arm64 +FROM base-${TARGETARCH} AS runner + +RUN apt update && apt upgrade -y && apt install -y \ + xz-utils autoconf libtool automake pkgconf bison flex + +# this is where the libraries will be installed and subsequently where the LIBS and INCLUDES can be found +ARG INSTALL_PREFIX=/install/libs +ENV INSTALL_PREFIX=${INSTALL_PREFIX} + +ARG LIB_EXPORTS_DIR=/install/exports +ENV LIB_EXPORTS_DIR=${LIB_EXPORTS_DIR} + +COPY --from=builder /install/libs ${INSTALL_PREFIX} +COPY --from=builder /install/exports ${LIB_EXPORTS_DIR} + +# needed in building pglite.wasm +ENV LLVM_NM=/emsdk/upstream/bin/llvm-nm \ No newline at end of file diff --git a/pglite-wasm/excluded.pglite.imports b/pglite-wasm/excluded.pglite.imports new file mode 100644 index 0000000000000..e0d801f2348b5 --- /dev/null +++ b/pglite-wasm/excluded.pglite.imports @@ -0,0 +1,32 @@ +__indirect_function_table +invoke_di +invoke_didi +invoke_dii +invoke_i +invoke_ii +invoke_iii +invoke_iiii +invoke_iiiii +invoke_iiiiii +invoke_iiiiiii +invoke_iiiiiiii +invoke_iij +invoke_ij +invoke_ijj +invoke_j +invoke_ji +invoke_jiiiii +invoke_v +invoke_vi +invoke_vii +invoke_viii +invoke_viiidi +invoke_viiii +invoke_viiiii +invoke_viiiiii +invoke_viiiiii +invoke_viiiiiiii +invoke_viiiiiiiii +invoke_viiji +invoke_vij +invoke_vijiiidjiiii diff --git a/pglite-wasm/included.pglite.imports b/pglite-wasm/included.pglite.imports new file mode 100644 index 0000000000000..63cbc5e039b90 --- /dev/null +++ b/pglite-wasm/included.pglite.imports @@ -0,0 +1,38 @@ +close +fcntl +free +get_buffer_addr +get_buffer_size +get_channel +getpid +gettimeofday +gmtime +interactive_one +interactive_read +interactive_write +ioctl +isalnum +isxdigit +lowerstr +main +malloc +memcmp +memcpy +memset +nanosleep +open +pgl_backend +pgl_initdb +pgl_shutdown +rand +read +readstoplist +realloc +searchstoplist +socket +srand +strcmp +strftime +strlen +strtoul +use_wire \ No newline at end of file diff --git a/pglite-wasm/pg_main.c b/pglite-wasm/pg_main.c index 07c21cf1b5b24..a3b77ece6cae6 100644 --- a/pglite-wasm/pg_main.c +++ b/pglite-wasm/pg_main.c @@ -26,6 +26,10 @@ #include /* chdir */ #include /* mkdir */ +#if defined(__EMSCRIPTEN__) +#include +#endif + // globals @@ -482,7 +486,8 @@ __attribute__ ((export_name("pgl_backend"))) #else remove(IDB_PIPE_BOOT); #endif - stdin = fdopen(saved_stdin, "r"); + // tdrz: I've commented this out!!! + // stdin = fdopen(saved_stdin, "r"); PDEBUG("# 479: initdb faking shutdown to complete WAL/OID states"); pg_proc_exit(66); diff --git a/pglite/Makefile b/pglite/Makefile new file mode 100644 index 0000000000000..4f4ef50406dd8 --- /dev/null +++ b/pglite/Makefile @@ -0,0 +1,35 @@ +# Packages other extensions into individual tar.gz archives + +top_builddir = .. +include $(top_builddir)/src/Makefile.global + +SUBDIRS = \ + pg_ivm \ + vector + +prefix ?= /install/pglite +EXTENSIONS_BUILD_ROOT := /tmp/extensions/build +ARCHIVE_DIR := /install/pglite/extensions + +EXTENSIONS := $(SUBDIRS) + +# Default target: build tarballs for all contribs +dist: $(addsuffix .tar.gz,$(EXTENSIONS)) + +# Pattern rule: build $(EXT).tar.gz for each extra extension +%.tar.gz: + @echo "=== Staging $* ===" + rm -rf $(EXTENSIONS_BUILD_ROOT)/$* + bash -c 'mkdir -p $(EXTENSIONS_BUILD_ROOT)/$*/$(prefix)/{bin,lib,share/extension,share/doc,share/postgresql/extension,share/postgresql/tsearch_data,include}' + $(MAKE) -C $* install DESTDIR=$(EXTENSIONS_BUILD_ROOT)/$* + @echo "=== Packaging $* ===" + mkdir -p $(ARCHIVE_DIR) + cd $(EXTENSIONS_BUILD_ROOT)/$*/$(prefix) && \ + files=$$(find . -type f -o -type l | sed 's|^\./||') && \ + tar -czf $(ARCHIVE_DIR)/$*.tar.gz $$files +# tar -C $(EXTENSIONS_BUILD_ROOT)/$*/$(prefix) -czf $(ARCHIVE_DIR)/$*.tar.gz . + +.PHONY: dist + + +$(recurse) \ No newline at end of file diff --git a/pglite/pg_ivm b/pglite/pg_ivm new file mode 160000 index 0000000000000..f4b40e93a6047 --- /dev/null +++ b/pglite/pg_ivm @@ -0,0 +1 @@ +Subproject commit f4b40e93a60478a1ea9d69f0fd305452e5c00690 diff --git a/pglite/vector b/pglite/vector new file mode 160000 index 0000000000000..2627c5ff775ae --- /dev/null +++ b/pglite/vector @@ -0,0 +1 @@ +Subproject commit 2627c5ff775ae6d7aef0c430121ccf857842d2f2 diff --git a/src/Makefile.shlib b/src/Makefile.shlib index aed880df4b876..befb131d977c4 100644 --- a/src/Makefile.shlib +++ b/src/Makefile.shlib @@ -225,13 +225,14 @@ ifeq ($(SHLIB_EXPORTS),) endif ifeq ($(PORTNAME), emscripten) - LINK.shared = emsdk-shared + LINK.shared = $(COMPILER) -shared -sSIDE_MODULE=1 -Wno-unused-function ifdef soname # emscripten uses unversioned shared libraries shlib = $(shlib_bare) soname = $(shlib_bare) endif - BUILD.exports = ( echo '{ global:'; $(AWK) '/^[^\#]/ {printf "%s;\n",$$1}' $<; echo ' local: *; };' ) >$@ + BUILD.exports = ( $(AWK) '/^[^\#]/ {printf "%s\n",$$1}' $< ) | sort -u >$@ +# BUILD.exports = ( echo '{ global:'; $(AWK) '/^[^\#]/ {printf "%s;\n",$$1}' $<; echo ' local: *; };' ) >$@ exports_file = $(SHLIB_EXPORTS:%.txt=%.list) # ifneq (,$(exports_file)) # LINK.shared += -Wl,--version-script=$(exports_file) diff --git a/src/backend/Makefile b/src/backend/Makefile index dbc1d4a148f3a..a79301f587fc3 100644 --- a/src/backend/Makefile +++ b/src/backend/Makefile @@ -77,14 +77,55 @@ ifeq ($(PORTNAME), emscripten) AR ?= llvm-ar LIBPGCORE ?= $(top_builddir)/libpgcore.a LIBPG = $(top_builddir)/libpostgres.a +LIBPGMAIN = $(top_builddir)/libpgmain.a PGCORE = $(top_builddir)/src/common/libpgcommon_srv.a $(top_builddir)/src/port/libpgport_srv.a $(LIBPG) PGMAIN = main/main.o tcop/postgres.o +PGLITE_MAIN = pglite-wasm/pg_main.c +EXPORTED_RUNTIME_METHODS="MEMFS,IDBFS,FS,setValue,getValue,UTF8ToString,stringToNewUTF8,stringToUTF8OnStack" +PGROOT=/install/pglite +PGPRELOAD=--preload-file ${PGROOT}/share/postgresql@/tmp/pglite/share/postgresql --preload-file ${PGROOT}/lib/postgresql@/tmp/pglite/lib/postgresql --preload-file $(top_builddir)/other/password@/tmp/pglite/password --preload-file $(top_builddir)/other/PGPASSFILE@/home/web_user/.pgpass --preload-file $(top_builddir)/other/empty@/tmp/pglite/bin/postgres --preload-file $(top_builddir)/other/empty@/tmp/pglite/bin/initdb postgres: $(OBJS) - $(AR) rcs $(top_builddir)/libpgmain.a $(PGMAIN) + $(AR) rcs $(LIBPGMAIN) $(PGMAIN) $(AR) rcs $(LIBPG) $(filter-out $(PGMAIN),$(call expand_subsys,$(ONLYOBJS))) $(CC) -r -o $(top_builddir)/libpgcore.o -Wl,--whole-archive $(PGCORE) $(AR) rcs $(LIBPGCORE) $(top_builddir)/libpgcore.o - COPTS="$(LOPTS)" $(CC) $(MAIN_MODULE) $(CFLAGS) $(LDFLAGS) -o $@ $(LIBPGCORE) $(top_builddir)/libpgmain.a $(LIBS) + COPTS="$(LOPTS)" $(CC) $(MAIN_MODULE) $(CFLAGS) $(LDFLAGS) -o $@$(X) $(LIBPGCORE) $(LIBPGMAIN) $(LIBS) + +# Extensions use functions from the core PG. These need to be exported by the emscripten compiler. +# The following target gathers all extension imports + the default ones (included.pglite.imports), +# excludes the one in excluded.pglite.imports and adds a leading _ to each. +pglite-exported-functions: + $(MKDIR_P) '$(emscripten_imports_dir)' + cat $(top_builddir)/pglite-wasm/excluded.*.imports $(top_builddir)/src/interfaces/libpq/exports.list $(LIB_EXPORTS_DIR)/libossp-uuid.exports | sort -u > '$(top_builddir)/pglite-wasm/excluded.imports' + cat $(DESTDIR)$(emscripten_extension_imports_dir)/*.imports '$(top_builddir)/pglite-wasm/included.pglite.imports' | \ + sort -u | \ + grep -Fvx -f '$(top_builddir)/pglite-wasm/excluded.imports' | \ + sed 's/^/_/' \ + > '$(emscripten_imports_dir)/exported_functions.txt' + +# -sDYLINK_DEBUG=2 use this for debugging missing exported symbols (ex when an extension calls a pgcore function that hasn't been exported) +# PGLITE_CFLAGS is something like "-O2" (aka release version) or "-g -gsource-map --no-wasm-opt" (aka debug version) +pglite: pglite-exported-functions + $(CC) $(CFLAGS) $(LDFLAGS) -DPG_PREFIX=/tmp/pglite -DCMA_MB=12 -I$(top_builddir)/src/include -I$(top_builddir)/src/ -I$(top_builddir)/src/interfaces/libpq -o pglite.o -c $(top_builddir)/$(PGLITE_MAIN) -Wno-incompatible-pointer-types-discards-qualifiers + $(CC) \ + $(PGLITE_CFLAGS) \ + -fPIC -sWASM_BIGINT -sMIN_SAFARI_VERSION=150000 -D__PYDK__=1 -m32 -D_FILE_OFFSET_BITS=64 -sSUPPORT_LONGJMP=emscripten -mno-bulk-memory -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-extended-const -mno-atomics -mno-tail-call -mno-multivalue -mno-relaxed-simd -mno-simd128 -mno-multimemory -mno-exception-handling -Wno-unused-command-line-argument -Wno-unreachable-code-fallthrough -Wno-unused-function -Wno-invalid-noreturn -Wno-declaration-after-statement -Wno-invalid-noreturn \ + -DPYDK=1 -DPG_PREFIX=/tmp/pglite -DCMA_MB=12 -o pglite.html --shell-file $(top_builddir)/pglite-wasm/repl.html \ + $(PGPRELOAD) \ + -sGLOBAL_BASE=12MB -ferror-limit=1 \ + -sFORCE_FILESYSTEM=1 -sNO_EXIT_RUNTIME=1 -sENVIRONMENT=node,web,worker \ + -sMAIN_MODULE=2 \ + -sEXPORTED_FUNCTIONS=@$(emscripten_imports_dir)/exported_functions.txt \ + -sMODULARIZE=1 -sEXPORT_ES6=1 -sEXPORT_NAME=Module \ + -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sERROR_ON_UNDEFINED_SYMBOLS=0 \ + -sEXPORTED_RUNTIME_METHODS=$(EXPORTED_RUNTIME_METHODS) \ + -I./src/include -I./src/ -I./src/interfaces/libpq \ + -L/install/libs/lib -L$(top_builddir) -L$(top_builddir)/src/interfaces/libpq/ \ + pglite.o \ + -Ldict_snowball \ + -lxslt \ + -lpgcore \ + -lnodefs.js -lidbfs.js -lxml2 -lz endif ifeq ($(PORTNAME), wasi) @@ -214,6 +255,11 @@ ifeq ($(PORTNAME), win32) ifeq ($(MAKE_DLL), true) $(INSTALL_DATA) libpostgres.a '$(DESTDIR)$(libdir)/libpostgres.a' endif +endif +ifeq ($(PORTNAME), emscripten) + $(INSTALL_DATA) $(LIBPG) '$(DESTDIR)$(libdir)/libpostgres.a' + $(INSTALL_DATA) $(LIBPGCORE) '$(DESTDIR)$(libdir)/libpgcore.a' + $(INSTALL_DATA) $(LIBPGMAIN) '$(DESTDIR)$(libdir)/libpgmain.a' endif $(MAKE) -C catalog install-data $(MAKE) -C tsearch install-data @@ -235,6 +281,9 @@ ifeq ($(MAKE_EXPORTS), true) $(INSTALL_DATA) $(POSTGRES_IMP) '$(DESTDIR)$(pkglibdir)/$(POSTGRES_IMP)' $(INSTALL_PROGRAM) $(MKLDEXPORT) '$(DESTDIR)$(pgxsdir)/$(MKLDEXPORT_DIR)/mkldexport.sh' endif +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) postgres.wasm '$(DESTDIR)$(bindir)/postgres.wasm' +endif .PHONY: install-bin @@ -255,6 +304,16 @@ ifeq ($(MAKE_EXPORTS), true) $(MKDIR_P) '$(DESTDIR)$(pgxsdir)/$(MKLDEXPORT_DIR)' endif +ifeq ($(PORTNAME), emscripten) +install-pglite: pglite + $(INSTALL_PROGRAM) pglite.html '$(DESTDIR)$(bindir)/pglite.html' + $(INSTALL_PROGRAM) pglite.wasm '$(DESTDIR)$(bindir)/pglite.wasm' + $(INSTALL_PROGRAM) pglite.js '$(DESTDIR)$(bindir)/pglite.js' + $(INSTALL_PROGRAM) pglite.data '$(DESTDIR)$(bindir)/pglite.data' +endif + +.PHONY: install-pglite + ########################################################################## @@ -273,6 +332,16 @@ ifeq ($(PORTNAME), win32) ifeq ($(MAKE_DLL), true) rm -f '$(DESTDIR)$(libdir)/libpostgres.a' endif +endif +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(libdir)/libpostgres.a' + rm -f '$(DESTDIR)$(libdir)/libpgcore.a' + rm -f '$(DESTDIR)$(libdir)/libpgmain.a' + rm -f '$(DESTDIR)$(bindir)/postgres.wasm' + rm -f '$(DESTDIR)$(bindir)/pglite.wasm' + rm -f '$(DESTDIR)$(bindir)/pglite.html' + rm -f '$(DESTDIR)$(bindir)/pglite.js' + rm -f '$(DESTDIR)$(bindir)/pglite.data' endif $(MAKE) -C catalog uninstall-data $(MAKE) -C tsearch uninstall-data @@ -295,6 +364,12 @@ endif ifeq ($(PORTNAME), win32) rm -f postgres.dll libpostgres.a $(WIN32RES) endif +ifeq ($(PORTNAME), emscripten) + rm -f postgres.wasm libpostgres.a libpgcore.a libpgmain.a pglite.o pglite.html pglite.js pglite.wasm pglite.data +endif +ifeq ($(PORTNAME), emscripten) + rm -f '$(emscripten_imports_dir)/exported_functions.txt' +endif distclean: clean # generated by configure diff --git a/src/backend/libpq/pqcomm.c b/src/backend/libpq/pqcomm.c index ff5277a682852..47fc44bf618e5 100644 --- a/src/backend/libpq/pqcomm.c +++ b/src/backend/libpq/pqcomm.c @@ -80,6 +80,10 @@ #include "utils/guc_hooks.h" #include "utils/memutils.h" +#if defined(__EMSCRIPTEN__) +#include +#endif + /* * Cope with the various platform-specific ways to spell TCP keepalive socket * options. This doesn't cover Windows, which as usual does its own thing. diff --git a/src/bin/initdb/Makefile b/src/bin/initdb/Makefile index 031cc77c9d61b..a960ce6580188 100644 --- a/src/bin/initdb/Makefile +++ b/src/bin/initdb/Makefile @@ -47,16 +47,25 @@ localtime.c: % : $(top_srcdir)/src/timezone/% install: all installdirs $(INSTALL_PROGRAM) initdb$(X) '$(DESTDIR)$(bindir)/initdb$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) initdb.wasm '$(DESTDIR)$(bindir)/initdb.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/initdb$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/initdb.wasm' +endif clean distclean: rm -f initdb$(X) $(OBJS) localtime.c rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f initdb.wasm +endif # ensure that changes in datadir propagate into object file initdb.o: initdb.c $(top_builddir)/src/Makefile.global diff --git a/src/bin/pg_amcheck/Makefile b/src/bin/pg_amcheck/Makefile index f9488c447a887..765032692b22a 100644 --- a/src/bin/pg_amcheck/Makefile +++ b/src/bin/pg_amcheck/Makefile @@ -33,16 +33,25 @@ pg_amcheck: $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils install: all installdirs $(INSTALL_PROGRAM) pg_amcheck$(X) '$(DESTDIR)$(bindir)/pg_amcheck$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_amcheck.wasm '$(DESTDIR)$(bindir)/pg_amcheck.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/pg_amcheck$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_amcheck.wasm' +endif clean distclean: rm -f pg_amcheck$(X) $(OBJS) rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_amcheck.wasm +endif check: $(prove_check) diff --git a/src/bin/pg_archivecleanup/Makefile b/src/bin/pg_archivecleanup/Makefile index 93fd703f22591..aaeac7ffd3524 100644 --- a/src/bin/pg_archivecleanup/Makefile +++ b/src/bin/pg_archivecleanup/Makefile @@ -18,16 +18,25 @@ pg_archivecleanup: $(OBJS) | submake-libpgport install: all installdirs $(INSTALL_PROGRAM) pg_archivecleanup$(X) '$(DESTDIR)$(bindir)/pg_archivecleanup$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_archivecleanup.wasm '$(DESTDIR)$(bindir)/pg_archivecleanup.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/pg_archivecleanup$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_archivecleanup.wasm' +endif clean distclean: rm -f pg_archivecleanup$(X) $(OBJS) rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_archivecleanup.wasm +endif check: $(prove_check) diff --git a/src/bin/pg_basebackup/Makefile b/src/bin/pg_basebackup/Makefile index 26c53e473f560..9b5df3ca2ecd3 100644 --- a/src/bin/pg_basebackup/Makefile +++ b/src/bin/pg_basebackup/Makefile @@ -63,6 +63,12 @@ install: all installdirs $(INSTALL_PROGRAM) pg_createsubscriber$(X) '$(DESTDIR)$(bindir)/pg_createsubscriber$(X)' $(INSTALL_PROGRAM) pg_receivewal$(X) '$(DESTDIR)$(bindir)/pg_receivewal$(X)' $(INSTALL_PROGRAM) pg_recvlogical$(X) '$(DESTDIR)$(bindir)/pg_recvlogical$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_basebackup.wasm '$(DESTDIR)$(bindir)/pg_basebackup.wasm' + $(INSTALL_PROGRAM) pg_createsubscriber.wasm '$(DESTDIR)$(bindir)/pg_createsubscriber.wasm' + $(INSTALL_PROGRAM) pg_receivewal.wasm '$(DESTDIR)$(bindir)/pg_receivewal.wasm' + $(INSTALL_PROGRAM) pg_recvlogical.wasm '$(DESTDIR)$(bindir)/pg_recvlogical.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' @@ -72,12 +78,21 @@ uninstall: rm -f '$(DESTDIR)$(bindir)/pg_createsubscriber$(X)' rm -f '$(DESTDIR)$(bindir)/pg_receivewal$(X)' rm -f '$(DESTDIR)$(bindir)/pg_recvlogical$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_basebackup.wasm' + rm -f '$(DESTDIR)$(bindir)/pg_createsubscriber.wasm' + rm -f '$(DESTDIR)$(bindir)/pg_receivewal.wasm' + rm -f '$(DESTDIR)$(bindir)/pg_recvlogical.wasm' +endif clean distclean: rm -f pg_basebackup$(X) pg_createsubscriber$(X) pg_receivewal$(X) pg_recvlogical$(X) \ $(BBOBJS) pg_createsubscriber.o pg_receivewal.o pg_recvlogical.o \ $(OBJS) rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_basebackup.wasm pg_createsubscriber.wasm pg_receivewal.wasm pg_recvlogical.wasm +endif check: $(prove_check) diff --git a/src/bin/pg_checksums/Makefile b/src/bin/pg_checksums/Makefile index 31de5fb467344..f524f483fe2a7 100644 --- a/src/bin/pg_checksums/Makefile +++ b/src/bin/pg_checksums/Makefile @@ -29,16 +29,25 @@ pg_checksums: $(OBJS) | submake-libpgport install: all installdirs $(INSTALL_PROGRAM) pg_checksums$(X) '$(DESTDIR)$(bindir)/pg_checksums$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_checksums.wasm '$(DESTDIR)$(bindir)/pg_checksums.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/pg_checksums$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_checksums.wasm' +endif clean distclean: rm -f pg_checksums$(X) $(OBJS) rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_checksums.wasm +endif check: $(prove_check) diff --git a/src/bin/pg_combinebackup/Makefile b/src/bin/pg_combinebackup/Makefile index c3729755ba4ba..167ec4be960f0 100644 --- a/src/bin/pg_combinebackup/Makefile +++ b/src/bin/pg_combinebackup/Makefile @@ -35,16 +35,25 @@ pg_combinebackup: $(OBJS) | submake-libpgport submake-libpgfeutils install: all installdirs $(INSTALL_PROGRAM) pg_combinebackup$(X) '$(DESTDIR)$(bindir)/pg_combinebackup$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_combinebackup.wasm '$(DESTDIR)$(bindir)/pg_combinebackup.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/pg_combinebackup$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_combinebackup.wasm' +endif clean distclean maintainer-clean: rm -f pg_combinebackup$(X) $(OBJS) rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_combinebackup.wasm +endif check: $(prove_check) diff --git a/src/bin/pg_config/Makefile b/src/bin/pg_config/Makefile index c54f68c9daddd..3d3c44fa2b4b8 100644 --- a/src/bin/pg_config/Makefile +++ b/src/bin/pg_config/Makefile @@ -19,23 +19,50 @@ OBJS = \ $(WIN32RES) \ pg_config.o +all: +ifeq ($(PORTNAME), emscripten) +all: pg_config pg_config_sh +else all: pg_config +endif pg_config: $(OBJS) | submake-libpgport $(CC) $(CFLAGS) $(OBJS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) +# the emscripten build generates multiple "runnable" files (.cjs/.mjs + .wasm) +# but other postgres tools expect exacly 'pg_config' as an executable +# the following rule creates a node script that calls the pg_config.cjs/.mjs +# pg_config.wasm is implicitly expected to be in the same folder +pg_config_sh: pg_config + echo "#!/usr/bin/env node" > pg_config + echo 'require("./pg_config$(X)")' >> pg_config + chmod +x pg_config + install: all installdirs $(INSTALL_SCRIPT) pg_config$(X) '$(DESTDIR)$(bindir)/pg_config$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_config.wasm '$(DESTDIR)$(bindir)/pg_config.wasm' + $(INSTALL_PROGRAM) pg_config '$(DESTDIR)$(bindir)/pg_config' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/pg_config$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_config.wasm' + rm -f '$(DESTDIR)$(bindir)/pg_config' +endif + clean distclean: rm -f pg_config$(X) $(OBJS) rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_config.wasm + rm -f pg_config +endif check: $(prove_check) diff --git a/src/bin/pg_controldata/Makefile b/src/bin/pg_controldata/Makefile index c3f64e189690a..0be4e65656b07 100644 --- a/src/bin/pg_controldata/Makefile +++ b/src/bin/pg_controldata/Makefile @@ -26,16 +26,25 @@ pg_controldata: $(OBJS) | submake-libpgport install: all installdirs $(INSTALL_PROGRAM) pg_controldata$(X) '$(DESTDIR)$(bindir)/pg_controldata$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_controldata.wasm '$(DESTDIR)$(bindir)/pg_controldata.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/pg_controldata$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_controldata.wasm' +endif clean distclean: rm -f pg_controldata$(X) $(OBJS) rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_controldata.wasm +endif check: $(prove_check) diff --git a/src/bin/pg_ctl/Makefile b/src/bin/pg_ctl/Makefile index a3fbad7a3a5a5..514cd8cf032f9 100644 --- a/src/bin/pg_ctl/Makefile +++ b/src/bin/pg_ctl/Makefile @@ -35,16 +35,25 @@ pg_ctl: $(OBJS) | submake-libpgport $(SUBMAKE_LIBPQ) install: all installdirs $(INSTALL_PROGRAM) pg_ctl$(X) '$(DESTDIR)$(bindir)/pg_ctl$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_ctl.wasm '$(DESTDIR)$(bindir)/pg_ctl.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/pg_ctl$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_ctl.wasm' +endif clean distclean: rm -f pg_ctl$(X) $(OBJS) rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_ctl.wasm +endif check: $(prove_check) diff --git a/src/bin/pg_dump/Makefile b/src/bin/pg_dump/Makefile index 930c741c95d85..cdd9cfcc5de96 100644 --- a/src/bin/pg_dump/Makefile +++ b/src/bin/pg_dump/Makefile @@ -57,6 +57,11 @@ install: all installdirs $(INSTALL_PROGRAM) pg_dump$(X) '$(DESTDIR)$(bindir)'/pg_dump$(X) $(INSTALL_PROGRAM) pg_restore$(X) '$(DESTDIR)$(bindir)'/pg_restore$(X) $(INSTALL_PROGRAM) pg_dumpall$(X) '$(DESTDIR)$(bindir)'/pg_dumpall$(X) +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_dump.wasm '$(DESTDIR)$(bindir)/pg_dump.wasm' + $(INSTALL_PROGRAM) pg_restore.wasm '$(DESTDIR)$(bindir)/pg_restore.wasm' + $(INSTALL_PROGRAM) pg_dumpall.wasm '$(DESTDIR)$(bindir)/pg_dumpall.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' @@ -69,7 +74,13 @@ installcheck: uninstall: rm -f $(addprefix '$(DESTDIR)$(bindir)'/, pg_dump$(X) pg_restore$(X) pg_dumpall$(X)) +ifeq ($(PORTNAME), emscripten) + rm -f $(addprefix '$(DESTDIR)$(bindir)'/, pg_dump.wasm pg_restore.wasm pg_dumpall.wasm) +endif clean distclean: rm -f pg_dump$(X) pg_restore$(X) pg_dumpall$(X) $(OBJS) pg_dump.o common.o pg_dump_sort.o pg_restore.o pg_dumpall.o rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_dump.wasm pg_restore.wasm pg_dumpall.wasm +endif \ No newline at end of file diff --git a/src/bin/pg_resetwal/Makefile b/src/bin/pg_resetwal/Makefile index 4228a5a772a9f..ff8e5d642a1f6 100644 --- a/src/bin/pg_resetwal/Makefile +++ b/src/bin/pg_resetwal/Makefile @@ -28,16 +28,25 @@ pg_resetwal: $(OBJS) | submake-libpgport install: all installdirs $(INSTALL_PROGRAM) pg_resetwal$(X) '$(DESTDIR)$(bindir)/pg_resetwal$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_resetwal.wasm '$(DESTDIR)$(bindir)/pg_resetwal.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/pg_resetwal$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_resetwal.wasm' +endif clean distclean: rm -f pg_resetwal$(X) $(OBJS) rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_resetwal.wasm +endif check: $(prove_check) diff --git a/src/bin/pg_rewind/Makefile b/src/bin/pg_rewind/Makefile index 12b138b2f2ce3..e0a5f5dd2c38f 100644 --- a/src/bin/pg_rewind/Makefile +++ b/src/bin/pg_rewind/Makefile @@ -42,16 +42,25 @@ xlogreader.c: % : $(top_srcdir)/src/backend/access/transam/% install: all installdirs $(INSTALL_PROGRAM) pg_rewind$(X) '$(DESTDIR)$(bindir)/pg_rewind$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_rewind.wasm '$(DESTDIR)$(bindir)/pg_rewind.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/pg_rewind$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_rewind.wasm' +endif clean distclean: rm -f pg_rewind$(X) $(OBJS) xlogreader.c rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_rewind.wasm +endif check: $(prove_check) diff --git a/src/bin/pg_test_fsync/Makefile b/src/bin/pg_test_fsync/Makefile index 4c5e518125033..e3bb3527a8858 100644 --- a/src/bin/pg_test_fsync/Makefile +++ b/src/bin/pg_test_fsync/Makefile @@ -18,6 +18,9 @@ pg_test_fsync: $(OBJS) | submake-libpgport install: all installdirs $(INSTALL_PROGRAM) pg_test_fsync$(X) '$(DESTDIR)$(bindir)/pg_test_fsync$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_test_fsync.wasm '$(DESTDIR)$(bindir)/pg_test_fsync.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' @@ -30,7 +33,13 @@ installcheck: uninstall: rm -f '$(DESTDIR)$(bindir)/pg_test_fsync$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_test_fsync.wasm' +endif clean distclean: rm -f pg_test_fsync$(X) $(OBJS) rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_test_fsync.wasm +endif \ No newline at end of file diff --git a/src/bin/pg_test_timing/Makefile b/src/bin/pg_test_timing/Makefile index 7f677edadb30f..eb4f95c2bf088 100644 --- a/src/bin/pg_test_timing/Makefile +++ b/src/bin/pg_test_timing/Makefile @@ -18,6 +18,9 @@ pg_test_timing: $(OBJS) | submake-libpgport install: all installdirs $(INSTALL_PROGRAM) pg_test_timing$(X) '$(DESTDIR)$(bindir)/pg_test_timing$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_test_timing.wasm '$(DESTDIR)$(bindir)/pg_test_timing.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' @@ -30,7 +33,13 @@ installcheck: uninstall: rm -f '$(DESTDIR)$(bindir)/pg_test_timing$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_test_timing.wasm' +endif clean distclean: rm -f pg_test_timing$(X) $(OBJS) rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_test_timing.wasm +endif \ No newline at end of file diff --git a/src/bin/pg_upgrade/Makefile b/src/bin/pg_upgrade/Makefile index bde91e2beb82c..faa309ac31691 100644 --- a/src/bin/pg_upgrade/Makefile +++ b/src/bin/pg_upgrade/Makefile @@ -42,17 +42,26 @@ pg_upgrade: $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils install: all installdirs $(INSTALL_PROGRAM) pg_upgrade$(X) '$(DESTDIR)$(bindir)/pg_upgrade$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_upgrade.wasm '$(DESTDIR)$(bindir)/pg_upgrade.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/pg_upgrade$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_upgrade.wasm' +endif clean distclean: rm -f pg_upgrade$(X) $(OBJS) rm -rf delete_old_cluster.sh log/ tmp_check/ \ reindex_hash.sql +ifeq ($(PORTNAME), emscripten) + rm -f pg_upgrade.wasm +endif export with_icu diff --git a/src/bin/pg_verifybackup/Makefile b/src/bin/pg_verifybackup/Makefile index 7c045f142e8d7..e49278f32d28c 100644 --- a/src/bin/pg_verifybackup/Makefile +++ b/src/bin/pg_verifybackup/Makefile @@ -30,16 +30,25 @@ pg_verifybackup: $(OBJS) | submake-libpq submake-libpgport submake-libpgfeutils install: all installdirs $(INSTALL_PROGRAM) pg_verifybackup$(X) '$(DESTDIR)$(bindir)/pg_verifybackup$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_verifybackup.wasm '$(DESTDIR)$(bindir)/pg_verifybackup.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/pg_verifybackup$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_verifybackup.wasm' +endif clean distclean: rm -f pg_verifybackup$(X) $(OBJS) rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_verifybackup.wasm +endif check: $(prove_check) diff --git a/src/bin/pg_waldump/Makefile b/src/bin/pg_waldump/Makefile index 4c1ee649501f4..1f08027439af9 100644 --- a/src/bin/pg_waldump/Makefile +++ b/src/bin/pg_waldump/Makefile @@ -38,16 +38,25 @@ $(RMGRDESCSOURCES): % : $(top_srcdir)/src/backend/access/rmgrdesc/% install: all installdirs $(INSTALL_PROGRAM) pg_waldump$(X) '$(DESTDIR)$(bindir)/pg_waldump$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_waldump.wasm '$(DESTDIR)$(bindir)/pg_waldump.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/pg_waldump$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_waldump.wasm' +endif clean distclean: rm -f pg_waldump$(X) $(OBJS) $(RMGRDESCSOURCES) xlogreader.c xlogstats.c rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_waldump.wasm +endif check: $(prove_check) diff --git a/src/bin/pg_walsummary/Makefile b/src/bin/pg_walsummary/Makefile index 1886c39e98b19..c686d9450641a 100644 --- a/src/bin/pg_walsummary/Makefile +++ b/src/bin/pg_walsummary/Makefile @@ -31,16 +31,25 @@ pg_walsummary: $(OBJS) | submake-libpgport submake-libpgfeutils install: all installdirs $(INSTALL_PROGRAM) pg_walsummary$(X) '$(DESTDIR)$(bindir)/pg_walsummary$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pg_walsummary.wasm '$(DESTDIR)$(bindir)/pg_walsummary.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/pg_walsummary$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pg_walsummary.wasm' +endif clean distclean maintainer-clean: rm -f pg_walsummary$(X) $(OBJS) rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f pg_walsummary.wasm +endif check: $(prove_check) diff --git a/src/bin/pgbench/Makefile b/src/bin/pgbench/Makefile index 987bf64df9de0..604f9493b35af 100644 --- a/src/bin/pgbench/Makefile +++ b/src/bin/pgbench/Makefile @@ -40,17 +40,26 @@ exprparse.o exprscan.o: exprparse.h install: all installdirs $(INSTALL_PROGRAM) pgbench$(X) '$(DESTDIR)$(bindir)/pgbench$(X)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) pgbench.wasm '$(DESTDIR)$(bindir)/pgbench.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/pgbench$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/pgbench.wasm' +endif clean distclean: rm -f pgbench$(X) $(OBJS) rm -rf tmp_check rm -f exprparse.h exprparse.c exprscan.c +ifeq ($(PORTNAME), emscripten) + rm -f pgbench.wasm +endif check: $(prove_check) diff --git a/src/bin/psql/Makefile b/src/bin/psql/Makefile index 374c4c3ab8f83..9c17566be2f2f 100644 --- a/src/bin/psql/Makefile +++ b/src/bin/psql/Makefile @@ -65,17 +65,26 @@ psqlscanslash.c: FLEX_FIX_WARNING=yes install: all installdirs $(INSTALL_PROGRAM) psql$(X) '$(DESTDIR)$(bindir)/psql$(X)' $(INSTALL_DATA) $(srcdir)/psqlrc.sample '$(DESTDIR)$(datadir)/psqlrc.sample' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) psql.wasm '$(DESTDIR)$(bindir)/psql.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' '$(DESTDIR)$(datadir)' uninstall: rm -f '$(DESTDIR)$(bindir)/psql$(X)' '$(DESTDIR)$(datadir)/psqlrc.sample' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/psql.wasm' +endif clean distclean: rm -f psql$(X) $(OBJS) lex.backup rm -rf tmp_check rm -f sql_help.h sql_help.c psqlscanslash.c +ifeq ($(PORTNAME), emscripten) + rm -f psql.wasm +endif check: $(prove_check) diff --git a/src/bin/scripts/Makefile b/src/bin/scripts/Makefile index 9633c99136880..f155e24f0c395 100644 --- a/src/bin/scripts/Makefile +++ b/src/bin/scripts/Makefile @@ -41,17 +41,33 @@ install: all installdirs $(INSTALL_PROGRAM) vacuumdb$(X) '$(DESTDIR)$(bindir)'/vacuumdb$(X) $(INSTALL_PROGRAM) reindexdb$(X) '$(DESTDIR)$(bindir)'/reindexdb$(X) $(INSTALL_PROGRAM) pg_isready$(X) '$(DESTDIR)$(bindir)'/pg_isready$(X) +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) createdb.wasm '$(DESTDIR)$(bindir)'/createdb.wasm + $(INSTALL_PROGRAM) dropdb.wasm '$(DESTDIR)$(bindir)'/dropdb.wasm + $(INSTALL_PROGRAM) createuser.wasm '$(DESTDIR)$(bindir)'/createuser.wasm + $(INSTALL_PROGRAM) dropuser.wasm '$(DESTDIR)$(bindir)'/dropuser.wasm + $(INSTALL_PROGRAM) clusterdb.wasm '$(DESTDIR)$(bindir)'/clusterdb.wasm + $(INSTALL_PROGRAM) vacuumdb.wasm '$(DESTDIR)$(bindir)'/vacuumdb.wasm + $(INSTALL_PROGRAM) reindexdb.wasm '$(DESTDIR)$(bindir)'/reindexdb.wasm + $(INSTALL_PROGRAM) pg_isready.wasm '$(DESTDIR)$(bindir)'/pg_isready.wasm +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f $(addprefix '$(DESTDIR)$(bindir)'/, $(addsuffix $(X), $(PROGRAMS))) +ifeq ($(PORTNAME), emscripten) + rm -f $(addprefix '$(DESTDIR)$(bindir)'/, $(addsuffix .wasm, $(PROGRAMS))) +endif clean distclean: rm -f $(addsuffix $(X), $(PROGRAMS)) $(addsuffix .o, $(PROGRAMS)) rm -f common.o $(WIN32RES) rm -rf tmp_check +ifeq ($(PORTNAME), emscripten) + rm -f $(addsuffix .wasm, $(PROGRAMS)) +endif export with_icu diff --git a/src/include/port/emscripten.h b/src/include/port/emscripten.h index db4903aa48f78..fb647c0d45199 100644 --- a/src/include/port/emscripten.h +++ b/src/include/port/emscripten.h @@ -4,10 +4,13 @@ #define I_WASM #if !defined(__cplusplus) -#include +// #include +// #include "/tmp/pglite/include/sdk_port.h" +// #include #endif -#include "/tmp/pglite/include/wasm_common.h" +// #include "/tmp/pglite/include/wasm_common.h" +#include "port/wasm_common.h" #define BOOT_END_MARK "build indices" diff --git a/src/include/port/pg_debug.h b/src/include/port/pg_debug.h new file mode 100644 index 0000000000000..73da07ddb9747 --- /dev/null +++ b/src/include/port/pg_debug.h @@ -0,0 +1,8 @@ +#ifndef I_PGDEBUG +#define I_PGDEBUG +#define WASM_USERNAME "postgres" +#define PDEBUG(string) +#define JSDEBUG(string) +#define ADEBUG(string) +#define PGDEBUG 0 +#endif diff --git a/src/include/port/sdk_port.h b/src/include/port/sdk_port.h new file mode 100644 index 0000000000000..7e6b178dce493 --- /dev/null +++ b/src/include/port/sdk_port.h @@ -0,0 +1,262 @@ +#if defined(__EMSCRIPTEN__) +#include + +#elif defined(__wasi__) + + +#ifndef I_WASI +#define I_WASI + +#undef HAVE_PTHREAD + +#if defined(HAVE_SETSID) +#undef HAVE_SETSID +#endif + +#if defined(HAVE_GETRLIMIT) +#undef HAVE_GETRLIMIT +#endif + + +#define PLATFORM_DEFAULT_SYNC_METHOD SYNC_METHOD_FDATASYNC + +#define EMSCRIPTEN_KEEPALIVE __attribute__((used)) +#define __declspec( dllimport ) __attribute__((used)) + +#define em_callback_func void +#define emscripten_set_main_loop(...) +#define emscripten_force_exit(...) +#define EM_JS(...) + +#include "wasm_common.h" + + +static pid_t +fork(void) { + puts("# 31: fork -1"); + return -1; +} + +// ======== signal ======================== +#define SA_RESTART 4 +#define SIG_SETMASK 2 + +#define SIG_BLOCK 0 +#define SIG_UNBLOCK 1 + +/* A signal handler. */ +typedef void (*handler_t) (int signal); +typedef unsigned char sigset_t; +typedef void (*__sighandler_t) (int); + +struct sigaction { + __sighandler_t sa_handler; + unsigned long sa_flags; +#ifdef SA_RESTORER + __sigrestore_t sa_restorer; +#endif + sigset_t sa_mask; +}; +extern int sigemptyset(sigset_t *set); +extern int sigaction(int signum, const struct sigaction *act, struct sigaction *oldact); +extern int sigdelset (sigset_t *set, int sig); +extern int sigfillset (sigset_t *set); +extern int sigprocmask (int operation, const sigset_t *set, sigset_t *old_set); +extern int sigaddset (sigset_t *set, int sig); + +// STUBS +extern int pthread_sigmask(int how, const sigset_t *set, sigset_t *oldset); +extern int sigismember(const sigset_t *set, int signum); +extern int sigpending(sigset_t *set); +extern int sigwait(const sigset_t *restrict set, int *restrict sig); + +// ==================================================== +// unistd +extern unsigned int alarm(unsigned int seconds); + +// ==================================================== + + +// WIP : + +#include +static uid_t +getuid(void) { + return 1000; +} + +static int +dup(int fd) { + puts("# 128: dup"); + return fd; +} +static int +dup2(int old, int new) { + puts("# 140: dup2"); + return -1; +} +static int +pipe(int fd[2]) { + puts("# 145: pipe"); + abort(); + return -1; +} + +#include +#define RLIMIT_NOFILE 7 +#define RLIMIT_STACK 3 +#define RLIM_INFINITY ((unsigned long int)(~0UL)) + +struct rlimit { + unsigned long rlim_cur; + unsigned long rlim_max; +}; +static int +getrlimit(int resource, struct rlimit *rlim) { + return -1; +} + + +static const char *gai_strerror_msg = "# 165: gai_strerror_msg"; +static const char * +gai_strerror(int errcode) { + return gai_strerror_msg; +} + + + + +// WIP: semaphores here +// ================================================================== +#include + +static int +semctl(int semid, int semnum, int cmd, ...) { + return 0; // -1; +} + +static int +semget(key_t key, int nsems, int semflg) { +#if 0 // PGDEBUG + printf("# 213: semget(key_t key = %d, int nsems=%d, int semflg=%d)\n", key, nsems, semflg); +#endif + return 1; +} + +static int +semop(int semid, struct sembuf *sops, size_t nsops) { + return 0; // -1; +} + + + +#include +#if defined(PYDK) +extern int shm_open(const char *name, int oflag, mode_t mode); +extern int shm_unlink(const char *name); +#else +static int +shm_open(const char *name, int oflag, mode_t mode) { + char tmpnam[128]; + int fd; + snprintf(tmpnam, 128, "/tmp%s", name); + fd=fileno(fopen(tmpnam, "w+")); + fprintf(stderr, "# 212: shm_open(%s) => %d\n", tmpnam, fd); + return fd; +} + +static int +shm_unlink(const char *name) { + char tmpnam[128]; + snprintf(tmpnam, 128, "/tmp%s", name); + fprintf(stderr, "# 220: shm_unlink(%s) STUB\n", tmpnam); + return remove(tmpnam); // -1 +} + +#endif + + + +#define system(command) system_wasi(command) +extern int system_wasi(const char *command); + + + +// time.h + +static void +tzset(void) { + puts("# 241: tzset(void) STUB"); +} + +#if defined(PG_INITDB) || defined(FE_UTILS_PRINT) || defined(PG_DUMP_PARALLEL) +static void +__SIG_IGN(int param) { +} +#endif + + +extern void sock_flush(); + + +// TODO: socket here +// ================================================================== + +/* +#include + +extern ssize_t sdk_recv(int sockfd, void *buf, size_t len, int flags); +extern ssize_t recvfrom(int socket, void *buffer, size_t length, int flags, void *address, socklen_t *address_len); + +extern ssize_t sdk_send(int sockfd, const void *buf, size_t len, int flags); +extern ssize_t sendto(int sockfd, const void *buf, size_t len, int flags, void *dest_addr, socklen_t addrlen); + +#define recv(sockfd, buf, len, flags) sdk_recv(sockfd, buf, len, flags) + + + + +static int +listen(int sockfd, int backlog) { + return 0; +} + +static struct group *_Nullable +getgrnam(const char *_Nullable name) { + return NULL; +} + + +static int +getaddrinfo(const char *restrict node, + const char *restrict service, + void *restrict hints, + void **restrict res) { + puts("# 60: getaddrinfo"); + return -1; +} +static void +freeaddrinfo(void *res) { + puts("# 65: freeaddrinfo"); +} + +extern ssize_t recvfrom_bc(int socket, void *buffer, size_t length, int flags, void *address, socklen_t *address_len); + +*/ + + +//#define pthread_mutex_lock(mut) sdk_pthread_mutex_lock(mut) +//extern int sdk_pthread_mutex_lock(void *mutex); + +/* + int pthread_mutex_lock(pthread_mutex_t *mutex); + int pthread_mutex_trylock(pthread_mutex_t *mutex); + int pthread_mutex_unlock(pthread_mutex_t *mutex); +*/ + + +#endif // I_WASI + +#else + #error "unknown port mode should be __EMSCRIPTEN__ or __wasi__" +#endif // __EMSCRIPTEN__ diff --git a/src/include/port/wasm_common.h b/src/include/port/wasm_common.h index 6f53f0fc5bb92..d8ade8cc7523e 100644 --- a/src/include/port/wasm_common.h +++ b/src/include/port/wasm_common.h @@ -45,7 +45,7 @@ # define PG_PLUGIN_INCLUDE "/pgdata/pg_plugin.h" #endif -#include "pg_debug.h" +#include "port/pg_debug.h" // #define COPY_INTERNAL #define COPY_OFF diff --git a/src/interfaces/ecpg/preproc/Makefile b/src/interfaces/ecpg/preproc/Makefile index 934b7cef1b0f5..ea5cbb1594caa 100644 --- a/src/interfaces/ecpg/preproc/Makefile +++ b/src/interfaces/ecpg/preproc/Makefile @@ -83,14 +83,23 @@ keywords.o: $(top_srcdir)/src/include/parser/kwlist.h install: all installdirs $(INSTALL_PROGRAM) ecpg$(X) '$(DESTDIR)$(bindir)' +ifeq ($(PORTNAME), emscripten) + $(INSTALL_PROGRAM) ecpg.wasm '$(DESTDIR)$(bindir)/ecpg.wasm' +endif installdirs: $(MKDIR_P) '$(DESTDIR)$(bindir)' uninstall: rm -f '$(DESTDIR)$(bindir)/ecpg$(X)' +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(bindir)/ecpg.wasm' +endif clean distclean: rm -f *.o ecpg$(X) rm -f typename.c rm -f preproc.y preproc.c preproc.h pgc.c c_kwlist_d.h ecpg_kwlist_d.h +ifeq ($(PORTNAME), emscripten) + rm -f ecpg.wasm +endif \ No newline at end of file diff --git a/src/makefiles/Makefile.emscripten b/src/makefiles/Makefile.emscripten index 3d37d043ad8c4..72e4c10a9fba4 100644 --- a/src/makefiles/Makefile.emscripten +++ b/src/makefiles/Makefile.emscripten @@ -14,3 +14,9 @@ AROPT = crs # Rule for building a shared library from a single .o file %.so: %.o $(CC) $(CFLAGS) $< $(LDFLAGS) $(LDFLAGS_SL) -shared -o $@ + +emscripten_include_dir := $(pkgincludedir)/emscripten +emscripten_base_dir := $(emscripten_include_dir)/base +emscripten_imports_dir := $(emscripten_base_dir)/imports +emscripten_extension_dir := $(emscripten_include_dir)/extension +emscripten_extension_imports_dir := $(emscripten_extension_dir)/imports \ No newline at end of file diff --git a/src/makefiles/pgxs.mk b/src/makefiles/pgxs.mk index 0de3737e789b4..bf4916166cba1 100644 --- a/src/makefiles/pgxs.mk +++ b/src/makefiles/pgxs.mk @@ -249,6 +249,12 @@ ifdef MODULES ifeq ($(with_llvm), yes) $(foreach mod, $(MODULES), $(call install_llvm_module,$(mod),$(mod).bc)) endif # with_llvm +ifeq ($(PORTNAME), emscripten) + $(LLVM_NM) -u *.o | awk '{print $$2}' | sed '/^$$/d' | sort -u > undef.txt + $(LLVM_NM) *.o | awk '$$2 ~ /^[TDB]$$/ {print $$3}' | sed '/^$$/d' | sort -u > defs.txt + comm -23 undef.txt defs.txt > '$(emscripten_extension_imports_dir)/$(MODULES).imports' + rm -f undef.txt defs.txt +endif # PORTNAME=emscripten endif # MODULES ifdef DOCS ifdef docdir @@ -271,6 +277,12 @@ ifdef MODULE_big ifeq ($(with_llvm), yes) $(call install_llvm_module,$(MODULE_big),$(OBJS)) endif # with_llvm +ifeq ($(PORTNAME), emscripten) + $(LLVM_NM) -u $(OBJS) | awk '{print $$2}' | sed '/^$$/d' | sort -u > undef.txt + $(LLVM_NM) $(OBJS) | awk '$$2 ~ /^[TDB]$$/ {print $$3}' | sed '/^$$/d' | sort -u > defs.txt + comm -23 undef.txt defs.txt > '$(emscripten_extension_imports_dir)/$(MODULE_big).imports' + rm -f undef.txt defs.txt +endif # PORTNAME=emscripten install: install-lib endif # MODULE_big @@ -302,6 +314,9 @@ ifdef MODULE_big installdirs: installdirs-lib endif # MODULE_big +ifeq ($(PORTNAME), emscripten) + $(MKDIR_P) '$(DESTDIR)$(emscripten_extension_imports_dir)' +endif uninstall: ifneq (,$(EXTENSION)) @@ -318,6 +333,9 @@ ifdef MODULES ifeq ($(with_llvm), yes) $(foreach mod, $(MODULES), $(call uninstall_llvm_module,$(mod))) endif # with_llvm +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(emscripten_extension_imports_dir)/$(MODULES).imports' +endif endif # MODULES ifdef DOCS rm -f $(addprefix '$(DESTDIR)$(docdir)/$(docmoduledir)'/, $(DOCS)) @@ -340,6 +358,10 @@ ifeq ($(with_llvm), yes) $(call uninstall_llvm_module,$(MODULE_big)) endif # with_llvm +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(emscripten_extension_imports_dir)/$(MODULE_big).imports' +endif + uninstall: uninstall-lib endif # MODULE_big diff --git a/src/pl/plpgsql/src/Makefile b/src/pl/plpgsql/src/Makefile index 63cb96fae3efc..7bdf2cd3f1e51 100644 --- a/src/pl/plpgsql/src/Makefile +++ b/src/pl/plpgsql/src/Makefile @@ -53,11 +53,20 @@ install: all install-lib install-data install-headers installdirs: installdirs-lib $(MKDIR_P) '$(DESTDIR)$(datadir)/extension' $(MKDIR_P) '$(DESTDIR)$(includedir_server)' +ifeq ($(PORTNAME), emscripten) + $(MKDIR_P) '$(DESTDIR)$(emscripten_extension_imports_dir)' +endif uninstall: uninstall-lib uninstall-data uninstall-headers install-data: installdirs $(INSTALL_DATA) $(addprefix $(srcdir)/, $(DATA)) '$(DESTDIR)$(datadir)/extension/' +ifeq ($(PORTNAME), emscripten) + $(LLVM_NM) -u $(OBJS) | awk '{print $$2}' | sed '/^$$/d' | sort -u > undef.txt + $(LLVM_NM) $(OBJS) | awk '$$2 ~ /^[TDB]$$/ {print $$3}' | sed '/^$$/d' | sort -u > defs.txt + comm -23 undef.txt defs.txt > '$(DESTDIR)$(emscripten_extension_imports_dir)/$(NAME).imports' + rm -f undef.txt defs.txt +endif # The plpgsql.h header file is needed by instrumentation plugins install-headers: installdirs @@ -65,6 +74,9 @@ install-headers: installdirs uninstall-data: rm -f $(addprefix '$(DESTDIR)$(datadir)/extension'/, $(notdir $(DATA))) +ifeq ($(PORTNAME), emscripten) + rm -f '$(DESTDIR)$(emscripten_extension_imports_dir)/$(NAME).imports' +endif uninstall-headers: rm -f '$(DESTDIR)$(includedir_server)/plpgsql.h' diff --git a/src/test/Makefile b/src/test/Makefile index dbd3192874d33..03bb5ccd34dfe 100644 --- a/src/test/Makefile +++ b/src/test/Makefile @@ -7,6 +7,12 @@ # src/test/Makefile # #------------------------------------------------------------------------- +ifeq ($(PORTNAME), emscripten) +# edited for pglite +all: $(echo src/test and src/test/isolation skipped) +clean check installcheck all-src-recurse: all +install: all +else subdir = src/test top_builddir = ../.. @@ -49,3 +55,5 @@ $(call recurse,installcheck, $(installable_dirs)) $(call recurse,install, $(installable_dirs)) $(recurse_always) + +endif \ No newline at end of file diff --git a/src/test/isolation/Makefile b/src/test/isolation/Makefile index ade2256ed3aa7..e0094bcb662ff 100644 --- a/src/test/isolation/Makefile +++ b/src/test/isolation/Makefile @@ -1,6 +1,12 @@ # # Makefile for isolation tests # +ifeq ($(PORTNAME), emscripten) +# edited for pglite +all: $(echo src/test and src/test/isolation skipped) +clean check installcheck all-src-recurse: all +install: all +else PGFILEDESC = "pg_isolation_regress/isolationtester - multi-client test driver" PGAPPICON = win32 @@ -72,3 +78,5 @@ installcheck-prepared-txns: all temp-install check-prepared-txns: all temp-install $(pg_isolation_regress_check) --schedule=$(srcdir)/isolation_schedule prepared-transactions prepared-transactions-cic + +endif \ No newline at end of file diff --git a/src/timezone/Makefile b/src/timezone/Makefile index c85e831247a5a..d871b67582378 100644 --- a/src/timezone/Makefile +++ b/src/timezone/Makefile @@ -48,8 +48,12 @@ endif # but GNU make versions <= 3.78.1 or perhaps later have a bug # that causes a segfault; GNU make 3.81 or later fixes this. ifeq (,$(ZIC)) +ifeq ($(PORTNAME), emscripten) +ZIC= /usr/sbin/zic +else ZIC= ./zic endif +endif zic: $(ZICOBJS) | submake-libpgport $(CC) $(CFLAGS) $(ZICOBJS) $(LDFLAGS) $(LDFLAGS_EX) $(LIBS) -o $@$(X) @@ -77,3 +81,6 @@ endif clean distclean: rm -f zic$(X) $(ZICOBJS) abbrevs.txt +ifeq ($(PORTNAME), emscripten) + rm -f zic.wasm +endif \ No newline at end of file diff --git a/wasm-build/sdk_port.h b/wasm-build/include/sdk_port.h similarity index 100% rename from wasm-build/sdk_port.h rename to wasm-build/include/sdk_port.h diff --git a/wasm-builder/Dockerfile b/wasm-builder/Dockerfile deleted file mode 100644 index a806e5e098b16..0000000000000 --- a/wasm-builder/Dockerfile +++ /dev/null @@ -1,23 +0,0 @@ -FROM debian:12 AS build_sdk - -ARG PG_VERSION=17.5 -ARG SDK_VERSION=3.1.74.11bi-w-n -ARG DEBUG=false -ARG OBJDUMP=true -ARG PG_BRANCH=REL_17_5_WASM-pglite - -ENV \ - PG_VERSION=$PG_VERSION \ - SDK_VERSION=$SDK_VERSION \ - SDKROOT=/tmp/sdk \ - SYS_PYTHON=/usr/bin/python3 \ - DEBUG=$DEBUG \ - BUILDS=3.12 \ - EMFLAVOUR=3.1.74 - -RUN apt-get update && \ - apt-get install -y git wget lz4 bzip2 pv bash python3 build-essential libreadline-dev zlib1g-dev bison flex xz-utils perl - -RUN apt-get install -y clang autoconf wget curl lz4 lsb-release zlib1g-dev libssl-dev - -RUN bash -c "wget -qO- https://github.com/electric-sql/portable-sdk/releases/download/3.1.74.12.0/python3.13-wasm-sdk-debian12-$(arch).tar.lz4 | lz4 -d | tar -xvf -" diff --git a/wasm-builder/build-with-docker.sh b/wasm-builder/build-with-docker.sh deleted file mode 100755 index 6396781c40027..0000000000000 --- a/wasm-builder/build-with-docker.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash - -# we are using a custom emsdk to build pglite wasm -# this is available as a docker image under electricsql/pglite-builder -IMG_NAME=${IMG_NAME:-"electricsql/pglite-builder"} - -[ -f postgres-pglite/configure ] || ln -s . postgres-pglite - -export WORKSPACE=${GITHUB_WORKSPACE:-$(pwd)} - -# normally would default to /workspace but that may cause trouble with debug paths in some IDE -export DOCKER_WORKSPACE=${DOCKER_WORKSPACE:-$WORKSPACE} - -cd $(realpath ${WORKSPACE}/postgres-pglite) - -[ -f ${BUILD_CONFIG:-postgres-pglite}/.buildconfig ] && cp ${BUILD_CONFIG:-postgres-pglite}/.buildconfig .buildconfig -[ -f ./pglite/.buildconfig ] && cp ./pglite/.buildconfig .buildconfig - - -source .buildconfig - -cat .buildconfig - - -if echo $IMG_NAME|grep -q debian -then - IMG_NAME="debian" - IMG_TAG="12" - wget -q -Osdk.tar.lz4 \ - https://github.com/electric-sql/portable-sdk/releases/download/${SDK_VERSION}/python3.13-wasm-sdk-${IMG_NAME}${IMG_TAG}-$(arch).tar.lz4 -else - IMG_TAG="${PG_VERSION}_${SDK_VERSION}" -fi - - -mkdir -p dist/pglite dist/extensions-emsdk - -if echo -n $@|grep -q it$ -then - PROMPT="&& bash ) || bash" -else - PROMPT=")" -fi - -if $WASI -then - OUT_DIR=wasi -else - OUT_DIR=emscripten -fi - -if $DEBUG -then - MAP_OUT_DIRS="-v ${WORKSPACE}/postgres-pglite/out/${OUT_DIR}/build:/tmp/sdk/build:rw -v ${WORKSPACE}/postgres-pglite/out/${OUT_DIR}/pglite:/tmp/pglite:rw" -else - MAP_OUT_DIRS="" -fi - -docker run $@ \ - --rm \ - --env-file .buildconfig \ - -e DEBUG=${DEBUG:-false} \ - -e WASI=${WASI:-false} \ - --workdir=${DOCKER_WORKSPACE} \ - -v ${WORKSPACE}/postgres-pglite:${DOCKER_WORKSPACE}:rw \ - -v ${WORKSPACE}/postgres-pglite/dist:/tmp/sdk/dist:rw \ - $MAP_OUT_DIRS \ - $IMG_NAME:$IMG_TAG \ - bash --noprofile --rcfile ./docker_rc.sh -ci "( ./wasm-build.sh ${WHAT:-\"contrib extra\"} $PROMPT" From 6c55ae16d798b6ab96856b0876b48f7f1c44f071 Mon Sep 17 00:00:00 2001 From: Tudor Zaharia <37596026+tdrz@users.noreply.github.com> Date: Wed, 24 Sep 2025 10:59:13 +0200 Subject: [PATCH 07/11] Skip system call during dbinit * although syscalls are supported by emscripten, this is not what we (always) want. in this particular case, initdb is querying the server configuration by executing postgres with various parameters. when running on node, the calls end up trying to execute on the OS, which leads to a series of errors and undesirable behavior. instead, these syscalls should remain inside the wasm sandbox somehow. until we decide on a proper way to do this, the initdb querying of postgres server configs will just fail (which is the case anyhow atm). --- src/bin/initdb/initdb.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/bin/initdb/initdb.c b/src/bin/initdb/initdb.c index 9aa9b48a79325..eb4699868b44e 100644 --- a/src/bin/initdb/initdb.c +++ b/src/bin/initdb/initdb.c @@ -1245,7 +1245,14 @@ test_specific_config_settings(int test_conns, int test_buffs) DEVNULL, DEVNULL); fflush(NULL); + #if defined(__EMSCRIPTEN__) + // emscripten supports (some?) syscalls, but that's not actually what we want, because we want to remain inside the wasm sandbox + // so just return a dummy value until we decide how to handle syscalls. + // TODO: https://github.com/electric-sql/pglite/issues/798 + status = 123; + #else status = system(cmd.data); + #endif // #if defined(__EMSCRIPTEN__) termPQExpBuffer(&cmd); From 1a0bdab383a0572804a6577432da6865a5bec314 Mon Sep 17 00:00:00 2001 From: Tudor Zaharia <37596026+tdrz@users.noreply.github.com> Date: Tue, 14 Oct 2025 12:18:43 +0200 Subject: [PATCH 08/11] Tdrz/ccoderefactoring (#50) * some cleanup * do not run ./configure if not needed; specify portname in clean script * cleanup * remove startup_hacks * readd initdb requirements * add linker search path for backend * remove get_buffer_size and get_buffer_addr" * remove recv_password_packet in interactive_one * removed pglite-modpython.c; removed __PYDK__ * remove specifing MIN_SAFARI_VERSION since that is the default value anyway * moved some of the emscripten flags to bash script to allow a native build * docker run under current user * improved cleanup * readd recv_password_packet to interactive_one.c * remove -it when running docker * script fix * fix build-with-docker.sh script for GH actions; cleanup target for extensions * fix * cb data transport * cleanup * nothing * more cleanup * cleanup is_repl and others * sort included imports * add a pq_flush() at the end of interactive_one(). this is needed because of the new frontend <-> backend data transfer mechanism because before the write was not buffered. * more cleanup * remove repl.html * more cleanup * :) * remove things not working --------- Co-authored-by: tudor --- build-pglite.sh | 35 +- build-with-docker.sh | 2 +- clean-pglite.sh | 12 +- other/empty | 1 + pglite-wasm/builder/Dockerfile | 9 +- pglite-wasm/included.pglite.imports | 9 +- pglite-wasm/interactive_one.c | 427 +------- pglite-wasm/pg_main.c | 59 +- pglite-wasm/pg_proto.c | 1 - pglite-wasm/pgl_initdb.c | 5 - pglite-wasm/pgl_sjlj.c | 5 - pglite-wasm/pgl_stubs.h | 12 - pglite-wasm/pglite-modpython.c | 218 ---- pglite-wasm/repl.html | 1226 ---------------------- pglite/Makefile | 6 +- src/backend/Makefile | 24 +- src/backend/libpq/pqcomm.c | 184 +--- src/backend/tcop/postgres.c | 12 +- src/include/port/wasm_common.h | 23 - wasm-build/sdk_port-wasi/sdk_port-wasi.c | 105 -- 20 files changed, 182 insertions(+), 2193 deletions(-) delete mode 100644 pglite-wasm/pglite-modpython.c delete mode 100644 pglite-wasm/repl.html diff --git a/build-pglite.sh b/build-pglite.sh index 7fa05c3900d95..8fd4b25572087 100755 --- a/build-pglite.sh +++ b/build-pglite.sh @@ -21,8 +21,28 @@ fi echo "pglite: PGLITE_CFLAGS=$PGLITE_CFLAGS" +# run ./configure only if config.status is older than this file +# TODO: we should ALSO check if any of the PGLITE_CFLAGS have changed and trigger a ./configure if they did!!! +REF_FILE="build-pglite.sh" +CONFIG_STATUS="config.status" +RUN_CONFIGURE=false + +if [ ! -f "$CONFIG_STATUS" ]; then + echo "$CONFIG_STATUS does not exist, need to run ./configure" + RUN_CONFIGURE=true +elif [ "$REF_FILE" -nt "$CONFIG_STATUS" ]; then + echo "$CONFIG_STATUS is older than $REF_FILE. Need to run ./configure." + RUN_CONFIGURE=true +else + echo "$CONFIG_STATUS exists and is newer than $REF_FILE. ./configure will NOT be run." +fi + # Step 1: configure the project -LDFLAGS="-sWASM_BIGINT -sUSE_PTHREADS=0" CFLAGS="${PGLITE_CFLAGS} -sWASM_BIGINT -fpic -sENVIRONMENT=node,web,worker -sSUPPORT_LONGJMP=emscripten -DPYDK=1 -DCMA_MB=12 -Wno-declaration-after-statement -Wno-macro-redefined -Wno-unused-function -Wno-missing-prototypes -Wno-incompatible-pointer-types" emconfigure ./configure ac_cv_exeext=.cjs --disable-spinlocks --disable-largefile --without-llvm --without-pam --disable-largefile --with-openssl=no --without-readline --without-icu --with-includes=$INSTALL_PREFIX/include:$INSTALL_PREFIX/include/libxml2 --with-libraries=$INSTALL_PREFIX/lib --with-uuid=ossp --with-zlib --with-libxml --with-libxslt --with-template=emscripten --prefix=$INSTALL_FOLDER || { echo 'error: emconfigure failed' ; exit 11; } +if [ "$RUN_CONFIGURE" = true ]; then + LDFLAGS="-sWASM_BIGINT -sUSE_PTHREADS=0" CFLAGS="${PGLITE_CFLAGS} -sWASM_BIGINT -fpic -sENVIRONMENT=node,web,worker -sSUPPORT_LONGJMP=emscripten -Wno-declaration-after-statement -Wno-macro-redefined -Wno-unused-function -Wno-missing-prototypes -Wno-incompatible-pointer-types" emconfigure ./configure ac_cv_exeext=.cjs --disable-spinlocks --disable-largefile --without-llvm --without-pam --disable-largefile --with-openssl=no --without-readline --without-icu --with-includes=$INSTALL_PREFIX/include:$INSTALL_PREFIX/include/libxml2 --with-libraries=$INSTALL_PREFIX/lib --with-uuid=ossp --with-zlib --with-libxml --with-libxslt --with-template=emscripten --prefix=$INSTALL_FOLDER || { echo 'error: emconfigure failed' ; exit 11; } +else + echo "Warning: configure has not been run because RUN_CONFIGURE=${RUN_CONFIGURE}" +fi # Step 2: make and install all except pglite emmake make PORTNAME=emscripten -j || { echo 'error: emmake make PORTNAME=emscripten -j' ; exit 21; } @@ -42,5 +62,16 @@ emmake make OPTFLAGS="" PORTNAME=emscripten LDFLAGS_SL="-sSIDE_MODULE=1" -C pgli PATH=$SAVE_PATH # Step 5: make and install pglite +# we define here "all" emscripten flags in order to allow native builds (like libpglite) +EXPORTED_RUNTIME_METHODS="MEMFS,IDBFS,FS,setValue,getValue,UTF8ToString,stringToNewUTF8,stringToUTF8OnStack,addFunction,removeFunction" +PGLITE_EMSCRIPTEN_FLAGS="-sWASM_BIGINT \ +-sSUPPORT_LONGJMP=emscripten \ +-sFORCE_FILESYSTEM=1 \ +-sNO_EXIT_RUNTIME=1 -sENVIRONMENT=node,web,worker \ +-sMAIN_MODULE=2 -sMODULARIZE=1 -sEXPORT_ES6=1 \ +-sEXPORT_NAME=Module -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH \ +-sERROR_ON_UNDEFINED_SYMBOLS=1 \ +-sEXPORTED_RUNTIME_METHODS=$EXPORTED_RUNTIME_METHODS" + # Building pglite itself needs to be the last step because of the PRELOAD_FILES parameter (a list of files and folders) need to be available. -PGLITE_CFLAGS=$PGLITE_CFLAGS emmake make PORTNAME=emscripten -j -C src/backend/ install-pglite || { echo 'emmake make OPTFLAGS="" PORTNAME=emscripten -j -C pglite' ; exit 51; } +PGLITE_CFLAGS="$PGLITE_CFLAGS $PGLITE_EMSCRIPTEN_FLAGS" emmake make PORTNAME=emscripten -j -C src/backend/ install-pglite || { echo 'emmake make OPTFLAGS="" PORTNAME=emscripten -j -C pglite' ; exit 51; } diff --git a/build-with-docker.sh b/build-with-docker.sh index d3d31a17fdc75..6a849b52ab863 100755 --- a/build-with-docker.sh +++ b/build-with-docker.sh @@ -8,6 +8,6 @@ docker run $@ \ --workdir=${DOCKER_WORKSPACE} \ -v .:${DOCKER_WORKSPACE}:rw \ -v ./dist:/install/pglite:rw \ - electricsql/pglite-builder:3.1.74_1 \ + electricsql/pglite-builder:3.1.74_2 \ ./build-pglite.sh diff --git a/clean-pglite.sh b/clean-pglite.sh index 3d6c86458baa6..4f72e3fa60a8b 100755 --- a/clean-pglite.sh +++ b/clean-pglite.sh @@ -1,6 +1,10 @@ #!/bin/bash -emmake make -C src/backend uninstall; emmake make -C src/backend clean; -emmake make -C pglite/ clean; emmake make -C pglite/ uninstall; -emmake make -C contrib/ clean; emmake make -C contrib/ uninstall; emmake make -C pglite clean; emmake make -C pglite uninstall; -emmake make clean; emmake make uninstal \ No newline at end of file +emmake make PORTNAME=emscripten -C src/backend uninstall; emmake make PORTNAME=emscripten -C src/backend clean; +emmake make PORTNAME=emscripten -C pglite/ clean; emmake make PORTNAME=emscripten -C pglite/ uninstall; +emmake make PORTNAME=emscripten -C contrib/ clean; emmake make PORTNAME=emscripten -C contrib/ uninstall; +emmake make PORTNAME=emscripten -C pglite clean; emmake make PORTNAME=emscripten -C pglite uninstall; +emmake make PORTNAME=emscripten clean; emmake make PORTNAME=emscripten uninstall; + +echo "removing config.status" +rm config.status \ No newline at end of file diff --git a/other/empty b/other/empty index e69de29bb2d1d..646d10863717a 100644 --- a/other/empty +++ b/other/empty @@ -0,0 +1 @@ +PGlite is the best! \ No newline at end of file diff --git a/pglite-wasm/builder/Dockerfile b/pglite-wasm/builder/Dockerfile index d8f5d931f75b1..cf613dd2ddc85 100644 --- a/pglite-wasm/builder/Dockerfile +++ b/pglite-wasm/builder/Dockerfile @@ -3,7 +3,7 @@ ARG BUILDPLATFORM # we only need to build on one platform, since we're interested in the WASM output # building on the native (BUILDPLATFORM) is much faster, so use that # remove "-arm64" suffix if building on x86_64 -FROM --platform=$BUILDPLATFORM emscripten/emsdk:${EMSDK_VER}-arm64 AS builder +FROM --platform=$BUILDPLATFORM emscripten/emsdk:${EMSDK_VER} AS builder ENV LLVM_NM=/emsdk/upstream/bin/llvm-nm @@ -21,7 +21,7 @@ WORKDIR /install/libs WORKDIR /src -# ENV EMCC_COMMON_FLAGS="-fPIC -sWASM_BIGINT -sMIN_SAFARI_VERSION=150000 -D__PYDK__=1 -O2 -m32 -D_FILE_OFFSET_BITS=64 -sSUPPORT_LONGJMP=emscripten -mno-bulk-memory -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-extended-const -mno-atomics -mno-tail-call -mno-multivalue -mno-relaxed-simd -mno-simd128 -mno-multimemory -mno-exception-handling -Wno-unused-command-line-argument -Wno-unreachable-code-fallthrough -Wno-unused-function -Wno-invalid-noreturn -Wno-declaration-after-statement -Wno-invalid-noreturn" +# ENV EMCC_COMMON_FLAGS="-fPIC -sWASM_BIGINT -sMIN_SAFARI_VERSION=150000 -O2 -m32 -D_FILE_OFFSET_BITS=64 -sSUPPORT_LONGJMP=emscripten -mno-bulk-memory -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-extended-const -mno-atomics -mno-tail-call -mno-multivalue -mno-relaxed-simd -mno-simd128 -mno-multimemory -mno-exception-handling -Wno-unused-command-line-argument -Wno-unreachable-code-fallthrough -Wno-unused-function -Wno-invalid-noreturn -Wno-declaration-after-statement -Wno-invalid-noreturn" ENV EMCC_COMMON_FLAGS="-O2 -fPIC" WORKDIR /src @@ -60,6 +60,8 @@ RUN ${LLVM_NM} /install/libs/lib/libssl.a | awk '$2 ~ /^[TDB]$/ {print $3}' | se WORKDIR /src RUN curl -L ftp://ftp.ossp.org/pkg/lib/uuid/uuid-1.6.2.tar.gz | tar xz +# COPY . . +# RUN tar xvf uuid-1.6.2.tar.gz WORKDIR /src/uuid-1.6.2 RUN CFLAGS="${EMCC_COMMON_FLAGS}" CXXFLAGS="${EMCC_COMMON_FLAGS}" emconfigure ./configure --build=aarch64-unknown-linux-gnu --enable-shared=no --enable-static=yes --with-perl=no --with-perl-compat=no --prefix=/install/libs --with-php=no --with-pic=yes RUN emmake make -j && emmake make install || true # install tries to strip the wasm, but it doesnt recognize the format, so ignore atm @@ -85,5 +87,8 @@ ENV LIB_EXPORTS_DIR=${LIB_EXPORTS_DIR} COPY --from=builder /install/libs ${INSTALL_PREFIX} COPY --from=builder /install/exports ${LIB_EXPORTS_DIR} +# allow access to anyone +RUN chmod -R 777 /install + # needed in building pglite.wasm ENV LLVM_NM=/emsdk/upstream/bin/llvm-nm \ No newline at end of file diff --git a/pglite-wasm/included.pglite.imports b/pglite-wasm/included.pglite.imports index 63cbc5e039b90..7a5d30d895111 100644 --- a/pglite-wasm/included.pglite.imports +++ b/pglite-wasm/included.pglite.imports @@ -1,15 +1,10 @@ close fcntl free -get_buffer_addr -get_buffer_size -get_channel getpid gettimeofday gmtime interactive_one -interactive_read -interactive_write ioctl isalnum isxdigit @@ -29,10 +24,10 @@ read readstoplist realloc searchstoplist +set_read_write_cbs socket srand strcmp strftime strlen -strtoul -use_wire \ No newline at end of file +strtoul \ No newline at end of file diff --git a/pglite-wasm/interactive_one.c b/pglite-wasm/interactive_one.c index 124e99c5059f4..2c2fa478c43ad 100644 --- a/pglite-wasm/interactive_one.c +++ b/pglite-wasm/interactive_one.c @@ -9,94 +9,32 @@ volatile sigjmp_buf local_sigjmp_buf; // track back how many ex raised in steps of the loop until sucessfull clear_error volatile int canary_ex = 0; -// track back mode used for last reply <0 socketfiles , 0== repl , > 0 cma addr -volatile int channel = 0; - -/* TODO : prevent multiple write and write while reading ? */ -volatile int cma_wsize = 0; -volatile int cma_rsize = 0; // also defined in postgres.c for pqcomm -volatile bool sockfiles = false; // also defined in postgres.c for pqcomm - -__attribute__((export_name("get_buffer_size"))) -int -get_buffer_size(int fd) { - return (CMA_MB * 1024 * 1024) / CMA_FD; -} - -// TODO add query size -__attribute__((export_name("get_buffer_addr"))) -int -get_buffer_addr(int fd) { - return 1 + ( get_buffer_size(fd) *fd); -} - -__attribute__((export_name("get_channel"))) -int -get_channel() { - return channel; -} - - -__attribute__((export_name("interactive_read"))) -int -interactive_read() { - /* should cma_rsize should be reset here ? */ - return cma_wsize; -} - - -static void pg_prompt() { - fprintf(stdout,"pg> %c\n", 4); +// read FROM JS +// (i guess return number of bytes written) +// ssize_t pglite_read(/* ignored */ int socket, void *buffer, size_t length,/* ignored */ int flags,/* ignored */ void *address,/* ignored */ socklen_t *address_len); +//typedef ssize_t (*pglite_read_t)(/* ignored */ int socket, void *buffer, size_t length,/* ignored */ int flags,/* ignored */ void *address,/* ignored */ unsigned int *address_len); +typedef ssize_t (*pglite_read_t)(void *buffer, size_t max_length); +extern pglite_read_t pglite_read; + +// write TO JS +// (i guess return number of bytes read) +// ssize_t pglite_write(/* ignored */ int sockfd, const void *buf, size_t len, /* ignored */ int flags); +// typedef ssize_t (*pglite_write_t)(/* ignored */ int sockfd, const void *buf, size_t len, /* ignored */ int flags); +typedef ssize_t (*pglite_write_t)(void *buffer, size_t length); +extern pglite_write_t pglite_write; + +__attribute__((export_name("set_read_write_cbs"))) +void +set_read_write_cbs(pglite_read_t read_cb, pglite_write_t write_cb) { + pglite_read = read_cb; + pglite_write = write_cb; } extern void AbortTransaction(void); extern void CleanupTransaction(void); extern void ClientAuthentication(Port *port); -extern FILE* SOCKET_FILE; -extern int SOCKET_DATA; - - - -/* -init sequence -___________________________________ -SubPostmasterMain / (forkexec) - InitPostmasterChild - shm attach - preload - - BackendInitialize(Port *port) -> collect initial packet - - pq_init(); - whereToSendOutput = DestRemote; - status = ProcessStartupPacket(port, false, false); - pq_startmsgread - pq_getbytes from pq_recvbuf - TODO: place PqRecvBuffer (8K) in lower mem for zero copy - - PerformAuthentication - ClientAuthentication(port) - CheckPasswordAuth SYNC!!!! ( sendAuthRequest flush -> recv_password_packet ) - InitShmemAccess/InitProcess/CreateSharedMemoryAndSemaphores - - BackendRun(port) - PostgresMain - - --> pq_flush() is synchronous - - -buffer sizes: - - https://github.com/postgres/postgres/blob/master/src/backend/libpq/pqcomm.c#L118 - - https://github.com/postgres/postgres/blob/master/src/common/stringinfo.c#L28 - - -*/ extern int ProcessStartupPacket(Port *port, bool ssl_done, bool gss_done); -extern void pq_recvbuf_fill(FILE* fp, int packetlen); #define PG_MAX_AUTH_TOKEN_LENGTH 65535 static char * @@ -187,44 +125,13 @@ static void io_init(bool in_auth, bool out_auth) { MyProcPort->canAcceptConnections = CAC_OK; #endif ClientAuthInProgress = out_auth; - SOCKET_FILE = NULL; - SOCKET_DATA = 0; PDEBUG("\n\n\n# 165: io_init --------- Ready for CLIENT ---------"); } volatile bool is_wire = true; -extern char * cma_port; extern void pq_startmsgread(void); -__attribute__((export_name("interactive_write"))) -void -interactive_write(int size) { - cma_rsize = size; - cma_wsize = 0; -} - -__attribute__((export_name("use_wire"))) -void -use_wire(int state) { -#if PGDEBUG - force_echo=true; -#endif - if (state>0) { -#if PGDEBUG - printf("\n\n# 194: PACKET START: wire mode, repl off, echo %d\n", force_echo); -#endif - is_wire = true; - is_repl = false; - } else { -#if PGDEBUG - printf("\n\n# 200: PACKET START: repl mode, no wire, echo %d\n", force_echo); -#endif - is_wire = false; - is_repl = true; - } -} - __attribute__((export_name("clear_error"))) void clear_error() { @@ -236,11 +143,11 @@ clear_error() { idle_in_transaction_timeout_enabled = false; idle_session_timeout_enabled = false; DoingCommandRead = false; -puts("# 239:" __FILE__ ); + pq_comm_reset(); EmitErrorReport(); debug_query_string = NULL; -puts("# 243:" __FILE__ ); + AbortCurrentTransaction(); if (am_walsender) @@ -279,16 +186,6 @@ puts("# 243:" __FILE__ ); send_ready_for_query = true; } -void discard_input(){ - if (!cma_rsize) - return; - pq_startmsgread(); - for (int i = 0; i < cma_rsize; i++) { - pq_getbyte(); - } - pq_endmsgread(); -} - void startup_auth() { /* code is in handshake/auth domain so read whole msg now */ @@ -301,7 +198,6 @@ startup_auth() { sf_connected++; PDEBUG("# 273: sending auth request"); //ClientAuthentication(MyProcPort); - discard_input(); ClientAuthInProgress = true; md5Salt[0]=0x01; @@ -339,7 +235,6 @@ startup_pass(bool check) { pfree(passwd); } else { PDEBUG("# 310: auth skip"); - discard_input(); } ClientAuthInProgress = false; @@ -365,24 +260,20 @@ PDEBUG("# 330: TODO: set a pgl started flag"); volatile int sf_connected = 0; } -extern void pg_startcma(); - __attribute__((export_name("interactive_one"))) void -interactive_one() { - int peek = -1; /* preview of firstchar with no pos change */ +interactive_one(int packetlen, int peek) { + // int peek = -1; /* preview of firstchar with no pos change */ int firstchar = 0; /* character read from getc() */ bool pipelining = true; StringInfoData input_message; StringInfoData *inBuf; FILE *stream ; FILE *fp = NULL; - int packetlen; + // int packetlen; bool had_notification = notifyInterruptPending; bool notified = false; // send_ready_for_query = false; -if (cma_rsize<0) - goto resume_on_error; if (!MyProcPort) { PDEBUG("# 353: client created"); @@ -390,31 +281,11 @@ if (cma_rsize<0) } #if PGDEBUG + puts("\n\n# 369: interactive_one"); if (notifyInterruptPending) PDEBUG("# 371: has notification !"); #endif - // this could be pg_flush in sync mode. - // but in fact we are writing socket data that was piled up previous frame async. - if (SOCKET_DATA>0) { - puts("# 361: ERROR flush after frame"); - goto wire_flush; - } - - if (!cma_rsize) { - // no cma : reading from file. writing to file. - if (!SOCKET_FILE) { - SOCKET_FILE = fopen(PGS_OLOCK, "w") ; - MyProcPort->sock = fileno(SOCKET_FILE); - } - } else { - // prepare file reply queue, just in case of cma overflow - // if unused the file will be kept open till next query. - if (!SOCKET_FILE) { - SOCKET_FILE = fopen(PGS_OLOCK, "w") ; - } - } - MemoryContextSwitchTo(MessageContext); MemoryContextReset(MessageContext); @@ -439,134 +310,21 @@ if (cma_rsize<0) } send_ready_for_query = false; } -// postgres.c 4627 - DoingCommandRead = true; - -#if defined(EMUL_CMA) - // temp fix for -O0 but less efficient than literal - #define IO ((char *)(1+(int)cma_port)) -#else - #define IO ((char *)(1)) -#endif - -/* - * in cma mode (cma_rsize>0), client call the wire loop itself waiting synchronously for the results - * in socketfiles mode, the wire loop polls a pseudo socket made from incoming and outgoing files. - * in repl mode (cma_rsize==0) output is on stdout not cma/socketfiles wire. - * repl mode is the simpleset mode where stdin is just copied into input buffer (limited by CMA size). - * TODO: allow to redirect stdout for fully external repl. - */ - - peek = IO[0]; - packetlen = cma_rsize; - - if (packetlen) { - sockfiles = false; - if (!is_repl) { - whereToSendOutput = DestRemote; - if (!is_wire) - PDEBUG("# 439: repl message in cma buffer !"); - } else { - if (is_wire) - PDEBUG("# 442: wire message in cma buffer for REPL !"); - whereToSendOutput = DestDebug; - } - } else { - fp = fopen(PGS_IN, "r"); -puts("# 475:" PGS_IN "\r\n"); - // read file in socket buffer for SocketBackend to consumme. - if (fp) { - fseek(fp, 0L, SEEK_END); - packetlen = ftell(fp); - if (packetlen) { - // set to always true if no REPL. -// is_wire = true; - resetStringInfo(inBuf); - rewind(fp); - /* peek on first char */ - peek = getc(fp); - rewind(fp); - if (is_repl && !is_wire) { - // sql in buffer - for (int i=0; i socketfiles -> repl + whereToSendOutput = DestRemote; #if PGDEBUG if (packetlen) IO[packetlen]=0; // wire blocks are not zero terminated - printf("\n# 524: fd=%d is_embed=%d is_repl=%d is_wire=%d fd %s,len=%d cma=%d peek=%d [%s]\n", MyProcPort->sock, is_embed, is_repl, is_wire, PGS_OLOCK, packetlen,cma_rsize, peek, IO); + printf("\n# 524: fd=%d is_embed=%d is_repl=%d is_wire=%d fd %s,len=%d peek=%d [%s]\n", MyProcPort->sock, is_embed, is_repl, is_wire, PGS_OLOCK, packetlen, peek, IO); #endif resetStringInfo(inBuf); - // when cma buffer is used to fake stdin, data is not read by socket/wire backend. - if (is_repl) { - for (int i=0; i0; +#if PGDEBUG + if (!pipelining) { + printf("# 556: end of wire, rfq=%d\n", send_ready_for_query); + } else { + printf("# 558: no end of wire -> pipelining, rfq=%d\n", send_ready_for_query); + } +#endif } else { /* nowire */ // pipelining = false; @@ -641,111 +395,34 @@ PDEBUG("# 507: NO DATA:" PGS_IN "\n"); if (pipelining) { pipelining = pq_buffer_remaining_data()>0; if (pipelining && send_ready_for_query) { +puts("# 631: PIPELINING + rfq"); ReadyForQuery(whereToSendOutput); send_ready_for_query = false; } } } -resume_on_error: - if (!is_repl) { -wire_flush: - if (!ClientAuthInProgress) { - /* process notifications (SYNC) */ - if (notifyInterruptPending) - ProcessNotifyInterrupt(false); - - if (send_ready_for_query) { -// PDEBUG("# 602: end packet - sending rfq\n"); - ReadyForQuery(DestRemote); - //done at postgres.c 4623 - send_ready_for_query = false; - } else { - PDEBUG("# 606: end packet - with no rfq\n"); - } - } else { - PDEBUG("# 609: end packet (ClientAuthInProgress - no rfq)\n"); - } - - if (SOCKET_DATA>0) { - if (sockfiles) { - channel = -1; - if (cma_wsize) { - puts("ERROR: cma was not flushed before socketfile interface"); - } - } else { - /* wsize may have increased with previous rfq so assign here */ - cma_wsize = SOCKET_DATA; - channel = cma_rsize + 2; - } - if (SOCKET_FILE) { - int outb = SOCKET_DATA; - fclose(SOCKET_FILE); - SOCKET_FILE = NULL; - SOCKET_DATA = 0; - - if (cma_wsize) { - PDEBUG("# 672: cma and sockfile ???\n"); - } - - if (sockfiles) { -#if PGDEBUG - printf("# 675: client:ready -> read(%d) " PGS_OLOCK "->" PGS_OUT"\n", outb); -#endif - rename(PGS_OLOCK, PGS_OUT); - } - } else { -#if PGDEBUG - printf("\n# 681: in[%d] out[%d] flushed\n", cma_rsize, cma_wsize); -#endif - SOCKET_DATA = 0; - } +wire_flush: + if (!ClientAuthInProgress) { + /* process notifications (SYNC) */ + if (notifyInterruptPending) + ProcessNotifyInterrupt(false); + if (send_ready_for_query) { + PDEBUG("# 602: end packet - sending rfq\n"); + ReadyForQuery(DestRemote); + //done at postgres.c 4623 + send_ready_for_query = false; } else { - cma_wsize = 0; - PDEBUG("# 698: no data, send empty ?"); -// TODO: dedup 739 - if (sockfiles) { - fclose(SOCKET_FILE); - SOCKET_FILE = NULL; - rename(PGS_OLOCK, PGS_OUT); - } + PDEBUG("# 606: end packet - with no rfq\n"); } } else { - pg_prompt(); -#if PGDEBUG - puts("# 683: repl output"); - if (SOCKET_DATA>0) { - puts("# 686: socket has data"); - if (sockfiles) - printf("# 688: socket file not flushed -> read(%d) " PGS_OLOCK "->" PGS_OUT"\n", SOCKET_DATA); - } else { -// TODO: dedup 723 - if (sockfiles) { - fclose(SOCKET_FILE); - SOCKET_FILE = NULL; - rename(PGS_OLOCK, PGS_OUT); - } - } - if (cma_wsize) - puts("ERROR: cma was not flushed before socketfile interface"); -#endif - } -return_early:; - /* always FD CLEANUP */ - if (fp) { - fclose(fp); - unlink(PGS_IN); + PDEBUG("# 609: end packet (ClientAuthInProgress - no rfq)\n"); } - - // always free kernel buffer !!! - cma_rsize = 0; - IO[0] = 0; - - #undef IO - +return_early:; // reset EX counter canary_ex = 0; + pq_flush(); } -#undef PGL_LOOP +#undef PGL_LOOP \ No newline at end of file diff --git a/pglite-wasm/pg_main.c b/pglite-wasm/pg_main.c index a3b77ece6cae6..bbfa698606dcb 100644 --- a/pglite-wasm/pg_main.c +++ b/pglite-wasm/pg_main.c @@ -45,7 +45,6 @@ volatile char *PGUSER; const char *progname; -volatile bool is_repl = true; volatile bool is_node = true; volatile bool is_embed = false; volatile int pgl_idb_status; @@ -62,7 +61,6 @@ volatile int async_restart = 1; #define WASM_PGDATA WASM_PREFIX "/base" -#define CMA_FD 1 extern bool IsPostmasterEnvironment; @@ -94,21 +92,8 @@ volatile bool send_ready_for_query = true; volatile bool idle_in_transaction_timeout_enabled = false; volatile bool idle_session_timeout_enabled = false; - -void -pg_free(void *ptr) { - free(ptr); -} - #include "../backend/tcop/postgres.c" - -// initdb + start on fd (pipe emulation) - - -static bool force_echo = false; - - #include "pgl_mains.c" #include "pgl_stubs.h" @@ -117,16 +102,11 @@ static bool force_echo = false; #include "pgl_initdb.c" - // interactive_one, heart of the async loop. - #include "./interactive_one.c" - static void main_pre(int argc, char *argv[]) { - - char key[256]; int i = 0; // extra env is always after normal args @@ -174,8 +154,8 @@ main_pre(int argc, char *argv[]) { #if defined(__EMSCRIPTEN__) EM_ASM( { Module.is_worker = (typeof WorkerGlobalScope !== 'undefined') && self instanceof WorkerGlobalScope; - Module.FD_BUFFER_MAX = $0; Module.emscripten_copy_to = console.warn;} - , (CMA_MB * 1024 * 1024) / CMA_FD); /* ( global mem start / num fd max ) */ + Module.FD_BUFFER_MAX = $0; Module.emscripten_copy_to = console.warn;} // tdrz: what is FD_BUFFER_MAX? + , (12 * 1024 * 1024)); /* ( global mem start / num fd max ) */ if (is_node) { setenv("ENVIRONMENT", "node", 1); @@ -194,7 +174,6 @@ main_pre(int argc, char *argv[]) { console.warn("prerun(C-web) worker=", Module.is_worker);} ); # endif - is_repl = true; } // *INDENT-OFF* EM_ASM({ @@ -563,35 +542,11 @@ PDEBUG("# 498: initdb faking shutdown to complete WAL/OID states in single mode" printf("# 550: argv0 (%s) PGUSER=%s PGDATA=%s\n PGDATABASE=%s REPL=%s\n", argv[0], PGUSER, PGDATA, getenv("PGDATABASE"), getenv("REPL")); #endif - progname = get_progname(argv[0]); - startup_hacks(progname); - g_argv = argv; - g_argc = argc; - - is_repl = strlen(getenv("REPL")) && getenv("REPL")[0] != 'N'; - is_embed = true; - - if (!is_repl) { - PDEBUG("# 562: exit with live runtime (nodb)"); - return 0; - } -#if defined(__wasi__) + progname = get_progname(argv[0]); + g_argv = argv; + g_argc = argc; + is_embed = true; - -#else - /* - main_post(); - - PDEBUG("# 565: repl"); - // so it is repl - main_repl(); - - if (is_node) { - PDEBUG("# 570: node repl"); - pg_repl_raf(); - } - */ - emscripten_force_exit(exit_code); -#endif + emscripten_force_exit(exit_code); return exit_code; } diff --git a/pglite-wasm/pg_proto.c b/pglite-wasm/pg_proto.c index fd21210f420d3..2ba14a846e5db 100644 --- a/pglite-wasm/pg_proto.c +++ b/pglite-wasm/pg_proto.c @@ -313,7 +313,6 @@ if (sf_connected) { PDEBUG("ERROR: more exits than connections"); } PDEBUG("# 251:proc_exit/skip and repl stop"); //proc_exit(0); - is_repl = false; ignore_till_sync = false; send_ready_for_query = false; diff --git a/pglite-wasm/pgl_initdb.c b/pglite-wasm/pgl_initdb.c index 19bf7fd405457..9536b257e7cea 100644 --- a/pglite-wasm/pgl_initdb.c +++ b/pglite-wasm/pgl_initdb.c @@ -49,10 +49,5 @@ pg_chmod(const char * path, int mode_t) { #include "bin/initdb/initdb.c" -void use_socketfile(void) { - is_repl = true; - is_embed = false; -} - diff --git a/pglite-wasm/pgl_sjlj.c b/pglite-wasm/pgl_sjlj.c index 6c05dafa9253f..469053a2705b2 100644 --- a/pglite-wasm/pgl_sjlj.c +++ b/pglite-wasm/pgl_sjlj.c @@ -57,11 +57,6 @@ #endif #if !defined(INITDB_SINGLE) -#if PGDEBUG - if (is_repl) - pg_prompt(); -#endif - if (pq_buffer_remaining_data()>0) { if (canary_ex++ > 8) abort(); diff --git a/pglite-wasm/pgl_stubs.h b/pglite-wasm/pgl_stubs.h index 2ad262c7f0096..5367add820685 100644 --- a/pglite-wasm/pgl_stubs.h +++ b/pglite-wasm/pgl_stubs.h @@ -14,8 +14,6 @@ // option_parse_int parse_sync_method #include "../src/fe_utils/option_utils.c" - - static void init_locale(const char *categoryname, int category, const char *locale) { if (pg_perm_setlocale(category, locale) == NULL && @@ -30,16 +28,7 @@ PostgresMain(const char *dbname, const char *username) { } -void -startup_hacks(const char *progname) { -#ifdef PG16 - SpinLockInit(&dummy_spinlock); -#endif -} - - // embedded initdb requirements - void get_restricted_token(void) { // stub @@ -76,7 +65,6 @@ pg_strdup(const char *in) { return tmp; } - char * simple_prompt(const char *prompt, bool echo) { return pg_strdup(""); diff --git a/pglite-wasm/pglite-modpython.c b/pglite-wasm/pglite-modpython.c deleted file mode 100644 index 8dc492eab99ad..0000000000000 --- a/pglite-wasm/pglite-modpython.c +++ /dev/null @@ -1,218 +0,0 @@ -#include -#include -#include - -#ifdef __wii__ -# include -# include -# include -#endif - -#include "w2c2/w2c2_base.h" -#include "wasi/wasi.h" -#include "${WASM2C}.h" - -void -trap(Trap trap) { - fprintf(stderr, "TRAP: %s\n", trapDescription(trap)); -#ifdef __wii__ - VIDEO_WaitVSync(); -#endif - abort(); -} - -U32 -wasi_snapshot_preview1__fd_renumber(void* ctx, U32 from, U32 to) { - return -1; -} - -wasmMemory* -wasiMemory(void* instance) { - return ${WASM2C}_memory((${WASM2C}Instance*)instance); -} - -extern char **environ; -volatile int skip_main = 0; -//char** __environ = NULL; -${WASM2C}Instance instance; - -#include "${WASM2C}.c" - -#define WASM_PREFIX "/tmp/pglite" -#define WASM_USERNAME "postgres" -#define PGDB WASM_PREFIX "/base" - - -char **g_argv; -int g_argc; - -char **copy_argv(int argc, char *argv[]) { - // calculate the contiguous argv buffer size - int length=0; - size_t ptr_args = argc + 1; - for (int i = 0; i < argc; i++) { - length += (strlen(argv[i]) + 1); - } - char** new_argv = (char**)malloc((ptr_args) * sizeof(char*) + length); - - // copy argv into the contiguous buffer - length = 0; - for (int i = 0; i < argc; i++) { - new_argv[i] = &(((char*)new_argv)[(ptr_args * sizeof(char*)) + length]); - strcpy(new_argv[i], argv[i]); - length += (strlen(argv[i]) + 1); - } - - // insert NULL terminating ptr at the end of the ptr array - new_argv[ptr_args-1] = NULL; - return (new_argv); -} - -int pre_main(int tmp_argc, char* tmp_argv[]) { - puts("# 84: pre_main"); - // __environ = environ; - - setenv("EMBED", "wasi", 1); - setenv("REPL", "N", 1); - - setenv("PGSYSCONFDIR", WASM_PREFIX, 1); - setenv("PGCLIENTENCODING", "UTF8", 1); - - setenv("TZ", "UTC", 1); - setenv("PGTZ", "UTC", 1); - setenv("PGUSER", WASM_USERNAME , 1); - setenv("PGDATA", PGDB , 1); - setenv("PGDATABASE", "template1" , 1); - setenv("PG_COLOR", "always", 0); - puts("# 87: global argc/argv"); - g_argv = copy_argv(tmp_argc, tmp_argv); - g_argc = sizeof(g_argv) / sizeof(char*) - 1; - - -# if defined(__MWERKS__) && defined(macintosh) - MaxApplZone(); - MoreMasters(); - MoreMasters(); - argc = ccommand(&argv); -# elif defined(__wii__) - /* TODO: get interactive console working */ - wiiInitVideo(); - fatInitDefault(); -# endif - - /* Initialize WASI */ - if (!wasiInit(g_argc, g_argv, environ)) { - fprintf(stderr, "failed to init WASI\n"); - return 1; - } - - if (!wasiFileDescriptorAdd(-1, "/", NULL)) { - fprintf(stderr, "failed to add preopen\n"); - return 1; - } - -# ifdef __MSL__ - SIOUXSetTitle("\p${WASM2C}"); -# endif - - ${WASM2C}Instantiate(&instance, NULL); - - return 0; -} - -// #define REACTOR - -#if defined(REACTOR) - #define STARTPROC(i) ${WASM2C}_setup(i) - // #define STARTPROC(i) ${WASM2C}_pg_initdb(i) -#else - #define STARTPROC(i) ${WASM2C}__start(&instance); -#endif - - -int main(int argc, char* argv[]); - -void do_main() { - puts("# 128: do_main Begin"); - STARTPROC(&instance); - puts("# 134: do_main End"); -} - -#if defined(__PYDK__) -# include "Python.h" - - static PyObject * ${WASM2C}_info(PyObject *self, PyObject *args, PyObject *kwds) - { - puts("${WASM2C} test function : return 42"); - return Py_BuildValue("i", 42); - } - - static PyObject * Begin(PyObject *self, PyObject *args, PyObject *kwds) { - puts("PyInit_${WASM2C}"); - - char *tmp_argv[] = { "/tmp/pglite/bin/postgres", "--single", "template1", NULL}; - int tmp_argc = sizeof(tmp_argv) / sizeof(char*) - 1; -puts("167"); - pre_main(tmp_argc, tmp_argv); -puts("169"); - -//${WASM2C}__start(&instance); - Py_RETURN_NONE; - } - - static PyObject * End(PyObject *self, PyObject *args, PyObject *kwds) { -puts("164: do main"); - do_main(); -puts("FREE"); - ${WASM2C}FreeInstance(&instance); - Py_RETURN_NONE; - } - -# include "${WASM2C}.pymod" - - static PyMethodDef mod_${WASM2C}_methods[] = { -#include "${WASM2C}.def" - {NULL, NULL, 0, NULL} - }; - - static struct PyModuleDef mod_${WASM2C} = { - PyModuleDef_HEAD_INIT, - "${WASM2C}", - NULL, - -1, - mod_${WASM2C}_methods, - NULL, // m_slots - NULL, // m_traverse - NULL, // m_clear - NULL, // m_free - }; - - - PyMODINIT_FUNC - PyInit_${WASM2C}(void) { - PyObject *${WASM2C}_mod = PyModule_Create(&mod_${WASM2C}); -# ifdef Py_GIL_DISABLED - PyUnstable_Module_SetGIL(${WASM2C}_mod, Py_MOD_GIL_NOT_USED); -# endif - return ${WASM2C}_mod; - } -#endif - - -int main(int argc, char* argv[]) { - if (!skip_main) { -puts("216"); - char *tmp_argv[] = { "/tmp/pglite/bin/postgres", "--single", "template1", NULL}; - int tmp_argc = sizeof(tmp_argv) / sizeof(char*) - 1; - skip_main = pre_main(tmp_argc, tmp_argv); -puts("220"); - if (!skip_main) { - do_main(); -puts("FREE"); - ${WASM2C}FreeInstance(&instance); - } - } else { - puts(" -- main skipped --"); - } - return skip_main; -} diff --git a/pglite-wasm/repl.html b/pglite-wasm/repl.html deleted file mode 100644 index 15e9ed2d20e1c..0000000000000 --- a/pglite-wasm/repl.html +++ /dev/null @@ -1,1226 +0,0 @@ - - - - - - - PG SHELL TEST - - - - - - - - - - - - - - - - - - -
-
emscripten
-
Downloading...
-
- -
- - - -
- - -
-pglite.wasi: Download file
-
-pglite.wasi + filesystem: Download file
-
-pg_dump.wasm: Download file
-
-
- -
- -
- -
- - - -#if MODULARIZE - - -#endif - - - - - diff --git a/pglite/Makefile b/pglite/Makefile index 4f4ef50406dd8..fc1f84c45bdc1 100644 --- a/pglite/Makefile +++ b/pglite/Makefile @@ -31,5 +31,9 @@ dist: $(addsuffix .tar.gz,$(EXTENSIONS)) .PHONY: dist +archive_clean: + @echo "=== Cleaning $* ===" + rm -rf $(EXTENSIONS_BUILD_ROOT)/$* + rm -rf $(ARCHIVE_DIR) -$(recurse) \ No newline at end of file +$(recurse) diff --git a/src/backend/Makefile b/src/backend/Makefile index a79301f587fc3..0bc6a84c3b10a 100644 --- a/src/backend/Makefile +++ b/src/backend/Makefile @@ -75,13 +75,12 @@ endif ifeq ($(PORTNAME), emscripten) AR ?= llvm-ar -LIBPGCORE ?= $(top_builddir)/libpgcore.a -LIBPG = $(top_builddir)/libpostgres.a -LIBPGMAIN = $(top_builddir)/libpgmain.a +LIBPGCORE ?= libpgcore.a +LIBPG = libpostgres.a +LIBPGMAIN = libpgmain.a PGCORE = $(top_builddir)/src/common/libpgcommon_srv.a $(top_builddir)/src/port/libpgport_srv.a $(LIBPG) PGMAIN = main/main.o tcop/postgres.o PGLITE_MAIN = pglite-wasm/pg_main.c -EXPORTED_RUNTIME_METHODS="MEMFS,IDBFS,FS,setValue,getValue,UTF8ToString,stringToNewUTF8,stringToUTF8OnStack" PGROOT=/install/pglite PGPRELOAD=--preload-file ${PGROOT}/share/postgresql@/tmp/pglite/share/postgresql --preload-file ${PGROOT}/lib/postgresql@/tmp/pglite/lib/postgresql --preload-file $(top_builddir)/other/password@/tmp/pglite/password --preload-file $(top_builddir)/other/PGPASSFILE@/home/web_user/.pgpass --preload-file $(top_builddir)/other/empty@/tmp/pglite/bin/postgres --preload-file $(top_builddir)/other/empty@/tmp/pglite/bin/initdb postgres: $(OBJS) @@ -105,22 +104,19 @@ pglite-exported-functions: # -sDYLINK_DEBUG=2 use this for debugging missing exported symbols (ex when an extension calls a pgcore function that hasn't been exported) # PGLITE_CFLAGS is something like "-O2" (aka release version) or "-g -gsource-map --no-wasm-opt" (aka debug version) +# PGLITE_EMSCRIPTEN_FLAGS are the emscripten flags to be passed to the build. Ideally they would be completely transparent, such that we +# could build a libpglite with or without those flags (to switch between native and wasm builds) pglite: pglite-exported-functions - $(CC) $(CFLAGS) $(LDFLAGS) -DPG_PREFIX=/tmp/pglite -DCMA_MB=12 -I$(top_builddir)/src/include -I$(top_builddir)/src/ -I$(top_builddir)/src/interfaces/libpq -o pglite.o -c $(top_builddir)/$(PGLITE_MAIN) -Wno-incompatible-pointer-types-discards-qualifiers + $(CC) $(CFLAGS) $(LDFLAGS) -DPG_PREFIX=/tmp/pglite -I$(top_builddir)/src/include -I$(top_builddir)/src/ -I$(top_builddir)/src/interfaces/libpq -o pglite.o -c $(top_builddir)/$(PGLITE_MAIN) -Wno-incompatible-pointer-types-discards-qualifiers $(CC) \ $(PGLITE_CFLAGS) \ - -fPIC -sWASM_BIGINT -sMIN_SAFARI_VERSION=150000 -D__PYDK__=1 -m32 -D_FILE_OFFSET_BITS=64 -sSUPPORT_LONGJMP=emscripten -mno-bulk-memory -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-extended-const -mno-atomics -mno-tail-call -mno-multivalue -mno-relaxed-simd -mno-simd128 -mno-multimemory -mno-exception-handling -Wno-unused-command-line-argument -Wno-unreachable-code-fallthrough -Wno-unused-function -Wno-invalid-noreturn -Wno-declaration-after-statement -Wno-invalid-noreturn \ - -DPYDK=1 -DPG_PREFIX=/tmp/pglite -DCMA_MB=12 -o pglite.html --shell-file $(top_builddir)/pglite-wasm/repl.html \ + -fPIC -m32 -D_FILE_OFFSET_BITS=64 -mno-bulk-memory -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-extended-const -mno-atomics -mno-tail-call -mno-multivalue -mno-relaxed-simd -mno-simd128 -mno-multimemory -mno-exception-handling -Wno-unused-command-line-argument -Wno-unreachable-code-fallthrough -Wno-unused-function -Wno-invalid-noreturn -Wno-declaration-after-statement -Wno-invalid-noreturn \ + -DPYDK=1 -DPG_PREFIX=/tmp/pglite -o pglite.html \ $(PGPRELOAD) \ - -sGLOBAL_BASE=12MB -ferror-limit=1 \ - -sFORCE_FILESYSTEM=1 -sNO_EXIT_RUNTIME=1 -sENVIRONMENT=node,web,worker \ - -sMAIN_MODULE=2 \ + -ferror-limit=1 \ -sEXPORTED_FUNCTIONS=@$(emscripten_imports_dir)/exported_functions.txt \ - -sMODULARIZE=1 -sEXPORT_ES6=1 -sEXPORT_NAME=Module \ - -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH -sERROR_ON_UNDEFINED_SYMBOLS=0 \ - -sEXPORTED_RUNTIME_METHODS=$(EXPORTED_RUNTIME_METHODS) \ -I./src/include -I./src/ -I./src/interfaces/libpq \ - -L/install/libs/lib -L$(top_builddir) -L$(top_builddir)/src/interfaces/libpq/ \ + -L/install/libs/lib -L$(top_builddir) -L$(top_builddir)/src/interfaces/libpq/ -L. \ pglite.o \ -Ldict_snowball \ -lxslt \ diff --git a/src/backend/libpq/pqcomm.c b/src/backend/libpq/pqcomm.c index 47fc44bf618e5..856e9d2a3623c 100644 --- a/src/backend/libpq/pqcomm.c +++ b/src/backend/libpq/pqcomm.c @@ -126,23 +126,56 @@ static char *PqSendBuffer; static int PqSendBufferSize; /* Size send buffer */ static size_t PqSendPointer; /* Next index to store a byte in PqSendBuffer */ static size_t PqSendStart; /* Next index to send a byte in PqSendBuffer */ -#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) static char PqRecvBuffer[PQ_RECV_BUFFER_SIZE]; static int PqRecvPointer; /* Next index to read a byte from PqRecvBuffer */ static int PqRecvLength; /* End of data available in PqRecvBuffer */ -#else -static char PqRecvBuffer_static[PQ_RECV_BUFFER_SIZE]; -static char *PqRecvBuffer; -static int PqRecvPointer; -static int PqRecvLength; +#if defined(__EMSCRIPTEN__) || defined(__wasi__) volatile int querylen = 0; volatile FILE* queryfp = NULL; -#endif -/* pglite specific */ -extern int cma_rsize; -extern bool sockfiles; +typedef ssize_t (*pglite_read_t)(void *buffer, size_t max_length); +extern pglite_read_t pglite_read; + +typedef ssize_t(*pglite_write_t)(void *buffer, size_t length); +extern pglite_write_t pglite_write; + +int EMSCRIPTEN_KEEPALIVE fcntl(int __fd, int __cmd, ...) { + // dummy + return 0; +} +int EMSCRIPTEN_KEEPALIVE setsockopt(int __fd, int __level, int __optname, + const void *__optval, socklen_t __optlen) { + // dummy + return 0; +} + +int EMSCRIPTEN_KEEPALIVE getsockopt(int __fd, int __level, int __optname, + void *__restrict __optval, + socklen_t *__restrict __optlen) { + // dummy + return 0; +} + +int EMSCRIPTEN_KEEPALIVE getsockname(int __fd, struct sockaddr * __addr, + socklen_t *__restrict __len) { + // dummy + return 0; + } + +ssize_t EMSCRIPTEN_KEEPALIVE + recv(int __fd, void *__buf, size_t __n, int __flags) { + ssize_t got = pglite_read(__buf, __n); + return got; + } + +ssize_t EMSCRIPTEN_KEEPALIVE + send(int __fd, const void *__buf, size_t __n, int __flags) { + ssize_t wrote = pglite_write(__buf, __n); + return wrote; + } + +#endif /* * Message status @@ -152,7 +185,6 @@ static bool PqCommReadingMsg; /* in the middle of reading a message */ /* Internal functions */ - static void socket_comm_reset(void); static void socket_close(int code, Datum arg); static void socket_set_nonblocking(bool nonblocking); @@ -166,6 +198,9 @@ static inline int internal_flush(void); static pg_noinline int internal_flush_buffer(const char *buf, size_t *start, size_t *end); +static int Lock_AF_UNIX(const char *unixSocketDir, const char *unixSocketPath); +static int Setup_AF_UNIX(const char *sock_path); + static const PQcommMethods PqCommSocketMethods = { .comm_reset = socket_comm_reset, .flush = socket_flush, @@ -175,10 +210,6 @@ static const PQcommMethods PqCommSocketMethods = { .putmessage_noblock = socket_putmessage_noblock }; -static int Lock_AF_UNIX(const char *unixSocketDir, const char *unixSocketPath); -static int Setup_AF_UNIX(const char *sock_path); - - const PQcommMethods *PqCommMethods = &PqCommSocketMethods; WaitEventSet *FeBeWaitSet; @@ -200,7 +231,6 @@ pq_init(ClientSocket *client_sock) port->sock = client_sock->sock; memcpy(&port->raddr.addr, &client_sock->raddr.addr, client_sock->raddr.salen); port->raddr.salen = client_sock->raddr.salen; -#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) /* fill in the server (local) address */ port->laddr.salen = sizeof(port->laddr.addr); if (getsockname(port->sock, @@ -292,7 +322,6 @@ pq_init(ClientSocket *client_sock) (void) pq_setkeepalivescount(tcp_keepalives_count, port); (void) pq_settcpusertimeout(tcp_user_timeout, port); } -#endif /* WASM */ /* initialize state variables */ PqSendBufferSize = PQ_SEND_BUFFER_SIZE; @@ -300,7 +329,6 @@ pq_init(ClientSocket *client_sock) PqSendPointer = PqSendStart = PqRecvPointer = PqRecvLength = 0; PqCommBusy = false; PqCommReadingMsg = false; -#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) /* set up process-exit hook to close the socket */ on_proc_exit(socket_close, 0); @@ -330,10 +358,6 @@ pq_init(ClientSocket *client_sock) MyLatch, NULL); AddWaitEventToSet(FeBeWaitSet, WL_POSTMASTER_DEATH, PGINVALID_SOCKET, NULL, NULL); -#else /* WASM */ - /* because we may fill before starting reading message */ - PqRecvBuffer = &PqRecvBuffer_static[0]; -#endif /* WASM */ /* * The event positions match the order we added them, but let's sanity * check them to be sure. @@ -753,7 +777,7 @@ Setup_AF_UNIX(const char *sock_path) Assert(Unix_socket_group); if (Unix_socket_group[0] != '\0') { -#if defined(WIN32) || defined(__wasi__) +#ifdef WIN32 elog(WARNING, "configuration item \"unix_socket_group\" is not supported on this platform"); #else char *endptr; @@ -932,20 +956,6 @@ pq_recvbuf(void) else PqRecvLength = PqRecvPointer = 0; } -#if defined(__EMSCRIPTEN__) || defined(__wasi__) - if (queryfp && querylen) { - int got = fread( PqRecvBuffer, 1, PQ_RECV_BUFFER_SIZE - PqRecvPointer, queryfp); - querylen -= got; - PqRecvLength += got; - if (querylen<=0) { - PDEBUG("# 931: could close fp early here " __FILE__); - queryfp = NULL; - } - if (got>0) - return 0; - } - return EOF; -#endif /* Ensure that we're in blocking mode */ socket_set_nonblocking(false); @@ -1048,9 +1058,7 @@ pq_getbyte_if_available(unsigned char *c) *c = PqRecvBuffer[PqRecvPointer++]; return 1; } -#if defined(__EMSCRIPTEN__) || (__wasi__) -puts("# 1044: pq_getbyte_if_available N/I in " __FILE__ ); abort(); -#else + /* Put the socket into non-blocking mode */ socket_set_nonblocking(true); @@ -1087,7 +1095,7 @@ puts("# 1044: pq_getbyte_if_available N/I in " __FILE__ ); abort(); /* EOF detected */ r = EOF; } -#endif + return r; } @@ -1154,7 +1162,6 @@ pq_discardbytes(size_t len) return 0; } - /* -------------------------------- * pq_buffer_remaining_data - return number of bytes in receive buffer * @@ -1176,26 +1183,6 @@ pq_buffer_remaining_data(void) * This must be called before any of the pq_get* functions. * -------------------------------- */ -#if defined(__EMSCRIPTEN__) || defined(__wasi__) -EMSCRIPTEN_KEEPALIVE void -pq_recvbuf_fill(FILE* fp, int packetlen) { - if (packetlen>PQ_RECV_BUFFER_SIZE) { - int got = fread( PqRecvBuffer, 1, PQ_RECV_BUFFER_SIZE, fp); - queryfp = fp; - querylen = packetlen - got; - PqRecvLength = got; - } else { - fread( PqRecvBuffer, packetlen, 1, fp); - PqRecvLength = packetlen; - queryfp = NULL; - querylen = 0; - } - PqRecvPointer = 0; -} - -#endif - -static char * PqSendBuffer_save; void pq_startmsgread(void) { @@ -1207,29 +1194,7 @@ pq_startmsgread(void) ereport(FATAL, (errcode(ERRCODE_PROTOCOL_VIOLATION), errmsg("terminating connection because protocol synchronization was lost"))); -#if defined(__EMSCRIPTEN__) || defined(__wasi__) - if (!pq_buffer_remaining_data()) { - if (sockfiles) { - PqRecvBuffer = &PqRecvBuffer_static[0]; - if (PqSendBuffer_save) - PqSendBuffer=PqSendBuffer_save; - PqSendBufferSize = PQ_SEND_BUFFER_SIZE; - } else { - PqRecvPointer = 0; - PqRecvLength = cma_rsize; - PqRecvBuffer = (char*)0x1; - - PqSendPointer = 0; - PqSendBuffer_save = PqSendBuffer; - PqSendBuffer = 2 + (char*)(cma_rsize); - PqSendBufferSize = (CMA_MB*1024*1024) - (int)(&PqSendBuffer[0]); - } - } -#if PDEBUG - printf("# 1225: pq_startmsgread cma_rsize=%d PqRecvLength=%d buf=%p reply=%p\n", cma_rsize, PqRecvLength, &PqRecvBuffer[0], &PqSendBuffer[0]); -#endif -#endif PqCommReadingMsg = true; } @@ -1352,56 +1317,7 @@ pq_getmessage(StringInfo s, int maxlen) return 0; } -#if defined(__EMSCRIPTEN__) || defined(__wasi__) -extern FILE* SOCKET_FILE; -extern int SOCKET_DATA; -static int -internal_putbytes(const char *s, size_t len) { - size_t amount; - if (!sockfiles) { - while (len > 0) { - /* If buffer is full, then flush it out from cma to file and continue from there */ - if (PqSendPointer >= PqSendBufferSize) { - int redirected = fwrite(PqSendBuffer, 1, PqSendPointer, SOCKET_FILE); - sockfiles = true; -#if PGDEBUG - fprintf(stderr, "# 1364: overflow %zu >= %d redirect=%d cma_rsize=%d CMA_MB=%d \n", PqSendPointer, PqSendBufferSize, redirected, cma_rsize, CMA_MB); -#endif - break; - } - amount = PqSendBufferSize - PqSendPointer; - if (amount > len) - amount = len; - memcpy(PqSendBuffer + PqSendPointer, s, amount); - PqSendPointer += amount; - s += amount; - len -= amount; - SOCKET_DATA+=amount; - } - } - - if (sockfiles) { - int wc= fwrite(s, 1, len, SOCKET_FILE); - SOCKET_DATA+=wc; - } - return 0; -} - -static int -socket_flush(void) { - return internal_flush(); -} - -static int -internal_flush(void) { - /* no flush for raw wire */ - if (sockfiles) { - PqSendStart = PqSendPointer = 0; - } - return 0; -} -#else static inline int internal_putbytes(const char *s, size_t len) @@ -1552,7 +1468,7 @@ internal_flush_buffer(const char *buf, size_t *start, size_t *end) *start = *end = 0; return 0; } -#endif /* wasm */ + /* -------------------------------- * pq_flush_if_writable - flush pending output if writable without blocking * diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index 72481df9bd16a..bbcf4112aead6 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -108,13 +108,13 @@ int client_connection_check_interval = 0; int restrict_nonsystem_relation_kind; #if (defined(__EMSCRIPTEN__) || defined(__wasi__)) -#if !defined(PGL_MAIN) - volatile int cma_rsize = 0; - volatile bool sockfiles = false; -#endif // PGL_MAIN bool quote_all_identifiers = false; -FILE* SOCKET_FILE = NULL; -int SOCKET_DATA = 0; + +typedef ssize_t (*pglite_read_t)(void *buffer, size_t max_length); +pglite_read_t pglite_read = NULL; + +typedef ssize_t(*pglite_write_t)(void *buffer, size_t length); +pglite_write_t pglite_write = NULL; #endif // WASM diff --git a/src/include/port/wasm_common.h b/src/include/port/wasm_common.h index d8ade8cc7523e..1801fb0468c01 100644 --- a/src/include/port/wasm_common.h +++ b/src/include/port/wasm_common.h @@ -102,30 +102,7 @@ extern int pg_valid_server_encoding_id_private(int encoding); #define proc_exit(arg) pg_proc_exit(arg) -/* -extern FILE* IDB_PIPE_FP; -extern int IDB_STAGE; -*/ -extern FILE* SOCKET_FILE; -extern int SOCKET_DATA; -extern int pgl_pclose(FILE *stream); -/* -#if !defined(PGL_MAIN) && !defined(PGL_INITDB_MAIN) -# if !defined(PG_EXEC) extern int pgl_pclose(FILE *stream); -#define pclose(stream) pg_pclose(stream) -# else -#if 1 -#include -int -pg_pclose(FILE *stream) { - puts("# 118:" __FILE__ " int pg_pclose(FILE *stream) STUB"); - return 0; -} -#endif -# endif // PG_EXEC -#endif // pgl -*/ /* * OpenPipeStream : another kind of pipe open in fd.c diff --git a/wasm-build/sdk_port-wasi/sdk_port-wasi.c b/wasm-build/sdk_port-wasi/sdk_port-wasi.c index 5dbab909264d7..f01d519d91075 100644 --- a/wasm-build/sdk_port-wasi/sdk_port-wasi.c +++ b/wasm-build/sdk_port-wasi/sdk_port-wasi.c @@ -316,11 +316,8 @@ int shm_unlink(const char *name) { #include // FILE+fprintf extern FILE* IDB_PIPE_FP; -extern FILE* SOCKET_FILE; -extern int SOCKET_DATA; extern int IDB_STAGE; - static inline int ends_with(const char *str, const char *suffix) { @@ -404,106 +401,4 @@ volatile int fd_sock = 0; volatile int fd_out = 2; volatile int fd_queue = 0; - - - -#if 0 -#define AF_UNIX 1 // PF_LOCAL - - - - -volatile bool web_warned = false; - -void sock_flush() { - if (fd_queue) { - printf(" -- 441 sockflush : AIO YIELD, expecting %s filled on return --\n", PGS_OUT); - if (!fd_FILE) { - if (!web_warned) { - puts("# 444: WARNING: fd_FILE not set but queue not empty, assuming web"); - web_warned = true; - } - - } else { - printf("# 449: SENT=%ld/%d fd_out=%d fno=%d\n", ftell(fd_FILE), fd_queue, fd_out, fileno(fd_FILE)); - fclose(fd_FILE); - rename(PGS_ILOCK, PGS_IN); -//freopen(PGS_ILOCK, "w+", fd_FILE); - fd_FILE = fopen(PGS_ILOCK, "w"); - fd_out = fileno(fd_FILE); - printf("# 455: fd_out=%d fno=%d\n", fd_out, fileno(fd_FILE)); - } - fd_queue = 0; - sched_yield(); - return; - } - - printf(" -- 462 sockflush[%d] : NO YIELD --\n",stage); - - // limit inf loops - if (stage++ > 1024) { - puts("# 466 sock_flush : busy looping ?"); - abort(); - } -} - - -volatile int fd_current_pos = 0; -volatile int fd_filesize = 0; - - -ssize_t recvfrom_bc(int socket, void *buffer, size_t length, int flags, void *address, socklen_t *address_len) { -// int busy = 0; - int rcv = -1; - sock_flush(); -/* - while (access(PGS_OUT, F_OK) != 0) { - if (!(++busy % 555111)) { - printf("# 471: FIXME: busy wait (%d) for input stream %s\n", busy, PGS_OUT); - } - if (busy>1665334) { - errno = EINTR; - return -1; - } - } -*/ - FILE *sock_in = fopen(PGS_OUT,"r"); - if (sock_in) { - if (!fd_filesize) { - fseek(sock_in, 0L, SEEK_END); - fd_filesize = ftell(sock_in); - } - fseek(sock_in, fd_current_pos, SEEK_SET); - - char *buf = buffer; - buf[0] = 0; - rcv = fread(buf, 1, length, sock_in); - - if (rcv Date: Mon, 3 Nov 2025 10:19:54 +0100 Subject: [PATCH 09/11] added backend for pgTap (#51) Co-authored-by: tudor --- .gitmodules | 3 +++ pglite/Makefile | 3 ++- pglite/pgtap | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) create mode 160000 pglite/pgtap diff --git a/.gitmodules b/.gitmodules index 9317a32956edc..94158709d174c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -4,3 +4,6 @@ [submodule "pglite/pgvector"] path = pglite/vector url = https://github.com/pgvector/pgvector.git +[submodule "pglite/pgtap"] + path = pglite/pgtap + url = https://github.com/theory/pgtap.git diff --git a/pglite/Makefile b/pglite/Makefile index fc1f84c45bdc1..f6bc99b8e50cc 100644 --- a/pglite/Makefile +++ b/pglite/Makefile @@ -5,7 +5,8 @@ include $(top_builddir)/src/Makefile.global SUBDIRS = \ pg_ivm \ - vector + vector \ + pgtap prefix ?= /install/pglite EXTENSIONS_BUILD_ROOT := /tmp/extensions/build diff --git a/pglite/pgtap b/pglite/pgtap new file mode 160000 index 0000000000000..6e3acbf4e0cb1 --- /dev/null +++ b/pglite/pgtap @@ -0,0 +1 @@ +Subproject commit 6e3acbf4e0cb19997edeafeabf1b18870ea68a5e From f5f1005d52e073c4e9fd7f67295bb7df7ac47552 Mon Sep 17 00:00:00 2001 From: Tudor Zaharia <37596026+tdrz@users.noreply.github.com> Date: Wed, 5 Nov 2025 12:07:14 +0100 Subject: [PATCH 10/11] Tdrz/backend pgdump (#52) Use emscripten built pg_dump. --------- Co-authored-by: tudor --- .gitignore | 3 + build-pglite.sh | 26 +++++++-- pglite-wasm/interactive_one.c | 25 -------- pglite-wasm/pgl_mains.c | 4 -- pglite-wasm/pgl_sjlj.c | 4 -- pglite-wasm/pgl_stubs.h | 11 +--- pglite/includes/pglite-comm.h | 88 +++++++++++++++++++++++++++++ src/backend/libpq/auth.c | 4 +- src/backend/libpq/be-fsstubs.c | 20 ++----- src/backend/libpq/be-secure.c | 4 ++ src/backend/libpq/pqcomm.c | 51 ----------------- src/backend/storage/ipc/latch.c | 2 - src/backend/tcop/postgres.c | 10 ---- src/backend/utils/error/elog.c | 5 -- src/backend/utils/fmgr/dfmgr.c | 5 +- src/backend/utils/misc/guc.c | 2 + src/bin/pg_config/Makefile | 12 ++-- src/bin/pg_config/run_pg_config.mjs | 12 ++++ src/bin/pg_dump/parallel.c | 4 +- src/bin/pg_dump/pg_backup_db.c | 2 +- src/bin/pg_dump/pg_dump.c | 32 ++--------- src/include/libpq/be-fsstubs.h | 6 +- src/interfaces/libpq/fe-connect.c | 35 +----------- src/interfaces/libpq/fe-secure.c | 4 ++ src/port/pqsignal.c | 9 --- 25 files changed, 159 insertions(+), 221 deletions(-) create mode 100644 pglite/includes/pglite-comm.h create mode 100644 src/bin/pg_config/run_pg_config.mjs diff --git a/.gitignore b/.gitignore index 2e67d7c8e6671..682db69d4d24c 100644 --- a/.gitignore +++ b/.gitignore @@ -46,7 +46,10 @@ lib*.pc /dist # emscripten build excludes +*.js +*.data *.cjs +pg_config.mjs *.wasm pglite.data pglite.js diff --git a/build-pglite.sh b/build-pglite.sh index 8fd4b25572087..3500a9dc1ebee 100755 --- a/build-pglite.sh +++ b/build-pglite.sh @@ -37,9 +37,25 @@ else echo "$CONFIG_STATUS exists and is newer than $REF_FILE. ./configure will NOT be run." fi +# we define here "all" emscripten flags in order to allow native builds (like libpglite) +EXPORTED_RUNTIME_METHODS="addFunction,removeFunction,FS,MEMFS" +PGLITE_EMSCRIPTEN_FLAGS="-sWASM_BIGINT \ +-sSUPPORT_LONGJMP=emscripten \ +-sFORCE_FILESYSTEM=1 \ +-sNO_EXIT_RUNTIME=0 -sENVIRONMENT=node,web,worker \ +-sMAIN_MODULE=2 -sMODULARIZE=1 -sEXPORT_ES6=1 \ +-sEXPORT_NAME=Module -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH \ +-sERROR_ON_UNDEFINED_SYMBOLS=0 \ +-sEXPORTED_RUNTIME_METHODS=$EXPORTED_RUNTIME_METHODS \ +-sTOTAL_MEMORY=32MB \ +--embed-file $(pwd)/other/PGPASSFILE@/home/web_user/.pgpass" + # Step 1: configure the project if [ "$RUN_CONFIGURE" = true ]; then - LDFLAGS="-sWASM_BIGINT -sUSE_PTHREADS=0" CFLAGS="${PGLITE_CFLAGS} -sWASM_BIGINT -fpic -sENVIRONMENT=node,web,worker -sSUPPORT_LONGJMP=emscripten -Wno-declaration-after-statement -Wno-macro-redefined -Wno-unused-function -Wno-missing-prototypes -Wno-incompatible-pointer-types" emconfigure ./configure ac_cv_exeext=.cjs --disable-spinlocks --disable-largefile --without-llvm --without-pam --disable-largefile --with-openssl=no --without-readline --without-icu --with-includes=$INSTALL_PREFIX/include:$INSTALL_PREFIX/include/libxml2 --with-libraries=$INSTALL_PREFIX/lib --with-uuid=ossp --with-zlib --with-libxml --with-libxslt --with-template=emscripten --prefix=$INSTALL_FOLDER || { echo 'error: emconfigure failed' ; exit 11; } + LDFLAGS="-sWASM_BIGINT -sUSE_PTHREADS=0" \ + LDFLAGS_SL="-sSIDE_MODULE=1" \ + LDFLAGS_EX=$PGLITE_EMSCRIPTEN_FLAGS \ + CFLAGS="${PGLITE_CFLAGS} -sWASM_BIGINT -fpic -sENVIRONMENT=node,web,worker -sSUPPORT_LONGJMP=emscripten -Wno-declaration-after-statement -Wno-macro-redefined -Wno-unused-function -Wno-missing-prototypes -Wno-incompatible-pointer-types" emconfigure ./configure ac_cv_exeext=.js --host aarch64-unknown-linux-gnu --disable-spinlocks --disable-largefile --without-llvm --without-pam --disable-largefile --with-openssl=no --without-readline --without-icu --with-includes=$INSTALL_PREFIX/include:$INSTALL_PREFIX/include/libxml2:$(pwd)/pglite/includes --with-libraries=$INSTALL_PREFIX/lib --with-uuid=ossp --with-zlib --with-libxml --with-libxslt --with-template=emscripten --prefix=$INSTALL_FOLDER || { echo 'error: emconfigure failed' ; exit 11; } else echo "Warning: configure has not been run because RUN_CONFIGURE=${RUN_CONFIGURE}" fi @@ -49,7 +65,7 @@ emmake make PORTNAME=emscripten -j || { echo 'error: emmake make PORTNAME=emscri emmake make PORTNAME=emscripten install || { echo 'error: emmake make PORTNAME=emscripten install' ; exit 22; } # Step 3.1: make all contrib extensions - do not install -emmake make PORTNAME=emscripten LDFLAGS_SL="-sSIDE_MODULE=1" -C contrib/ -j || { echo 'error: emmake make PORTNAME=emscripten -C contrib/ -j' ; exit 31; } +emmake make PORTNAME=emscripten -C contrib/ -j || { echo 'error: emmake make PORTNAME=emscripten -C contrib/ -j' ; exit 31; } # Step 3.2: make dist contrib extensions - this will create an archive for each extension emmake make PORTNAME=emscripten -C contrib/ dist || { echo 'error: emmake make PORTNAME=emscripten -C contrib/ dist' ; exit 32; } # the above will also create a file with the imports that each extension needs - we pass these as input in the next step for emscripten to keep alive @@ -58,11 +74,10 @@ emmake make PORTNAME=emscripten -C contrib/ dist || { echo 'error: emmake make P SAVE_PATH=$PATH PATH=$PATH:$INSTALL_FOLDER/bin emmake make OPTFLAGS="" PORTNAME=emscripten -j -C pglite || { echo 'error: emmake make OPTFLAGS="" PORTNAME=emscripten -j -C pglite' ; exit 41; } -emmake make OPTFLAGS="" PORTNAME=emscripten LDFLAGS_SL="-sSIDE_MODULE=1" -C pglite/ dist || { echo 'error: make OPTFLAGS="" PORTNAME=emscripten LDFLAGS_SL="-sSIDE_MODULE=1" -C pglite/ dist ' ; exit 42; } +emmake make OPTFLAGS="" PORTNAME=emscripten -C pglite/ dist || { echo 'error: make OPTFLAGS="" PORTNAME=emscripten -C pglite/ dist ' ; exit 42; } PATH=$SAVE_PATH # Step 5: make and install pglite -# we define here "all" emscripten flags in order to allow native builds (like libpglite) EXPORTED_RUNTIME_METHODS="MEMFS,IDBFS,FS,setValue,getValue,UTF8ToString,stringToNewUTF8,stringToUTF8OnStack,addFunction,removeFunction" PGLITE_EMSCRIPTEN_FLAGS="-sWASM_BIGINT \ -sSUPPORT_LONGJMP=emscripten \ @@ -70,8 +85,7 @@ PGLITE_EMSCRIPTEN_FLAGS="-sWASM_BIGINT \ -sNO_EXIT_RUNTIME=1 -sENVIRONMENT=node,web,worker \ -sMAIN_MODULE=2 -sMODULARIZE=1 -sEXPORT_ES6=1 \ -sEXPORT_NAME=Module -sALLOW_TABLE_GROWTH -sALLOW_MEMORY_GROWTH \ --sERROR_ON_UNDEFINED_SYMBOLS=1 \ +-sERROR_ON_UNDEFINED_SYMBOLS=0 \ -sEXPORTED_RUNTIME_METHODS=$EXPORTED_RUNTIME_METHODS" - # Building pglite itself needs to be the last step because of the PRELOAD_FILES parameter (a list of files and folders) need to be available. PGLITE_CFLAGS="$PGLITE_CFLAGS $PGLITE_EMSCRIPTEN_FLAGS" emmake make PORTNAME=emscripten -j -C src/backend/ install-pglite || { echo 'emmake make OPTFLAGS="" PORTNAME=emscripten -j -C pglite' ; exit 51; } diff --git a/pglite-wasm/interactive_one.c b/pglite-wasm/interactive_one.c index 2c2fa478c43ad..c520e2d149c74 100644 --- a/pglite-wasm/interactive_one.c +++ b/pglite-wasm/interactive_one.c @@ -1,35 +1,10 @@ #include // access, unlink #define PGL_LOOP -#if defined(__wasi__) -// volatile sigjmp_buf void*; -#else volatile sigjmp_buf local_sigjmp_buf; -#endif // track back how many ex raised in steps of the loop until sucessfull clear_error volatile int canary_ex = 0; -// read FROM JS -// (i guess return number of bytes written) -// ssize_t pglite_read(/* ignored */ int socket, void *buffer, size_t length,/* ignored */ int flags,/* ignored */ void *address,/* ignored */ socklen_t *address_len); -//typedef ssize_t (*pglite_read_t)(/* ignored */ int socket, void *buffer, size_t length,/* ignored */ int flags,/* ignored */ void *address,/* ignored */ unsigned int *address_len); -typedef ssize_t (*pglite_read_t)(void *buffer, size_t max_length); -extern pglite_read_t pglite_read; - -// write TO JS -// (i guess return number of bytes read) -// ssize_t pglite_write(/* ignored */ int sockfd, const void *buf, size_t len, /* ignored */ int flags); -// typedef ssize_t (*pglite_write_t)(/* ignored */ int sockfd, const void *buf, size_t len, /* ignored */ int flags); -typedef ssize_t (*pglite_write_t)(void *buffer, size_t length); -extern pglite_write_t pglite_write; - -__attribute__((export_name("set_read_write_cbs"))) -void -set_read_write_cbs(pglite_read_t read_cb, pglite_write_t write_cb) { - pglite_read = read_cb; - pglite_write = write_cb; -} - extern void AbortTransaction(void); extern void CleanupTransaction(void); extern void ClientAuthentication(Port *port); diff --git a/pglite-wasm/pgl_mains.c b/pglite-wasm/pgl_mains.c index 7dc200745a4d7..ab20394e07dff 100644 --- a/pglite-wasm/pgl_mains.c +++ b/pglite-wasm/pgl_mains.c @@ -218,13 +218,9 @@ PDEBUG("# 164:" __FILE__); initStringInfo(&row_description_buf); MemoryContextSwitchTo(TopMemoryContext); -#if defined(__wasi__) - puts("# 210: sjlj exception handler off in initdb-wasi"); -#else # define INITDB_SINGLE # include "pgl_sjlj.c" # undef INITDB_SINGLE -#endif // sjlj if (!ignore_till_sync) send_ready_for_query = true; /* initially, or after error */ diff --git a/pglite-wasm/pgl_sjlj.c b/pglite-wasm/pgl_sjlj.c index 469053a2705b2..56bdde5cf9755 100644 --- a/pglite-wasm/pgl_sjlj.c +++ b/pglite-wasm/pgl_sjlj.c @@ -1,6 +1,3 @@ -#if defined(__wasi__) - PDEBUG("# 2:" __FILE__ ": sjlj exception handler off"); -#else if (sigsetjmp(local_sigjmp_buf, 1) != 0) { #if !defined(INITDB_SINGLE) @@ -67,5 +64,4 @@ } PG_exception_stack = &local_sigjmp_buf; -#endif diff --git a/pglite-wasm/pgl_stubs.h b/pglite-wasm/pgl_stubs.h index 5367add820685..9a56a5973712e 100644 --- a/pglite-wasm/pgl_stubs.h +++ b/pglite-wasm/pgl_stubs.h @@ -1,15 +1,6 @@ #pragma once - -// wasi only stubs -#if defined(__wasi__) -# undef PQEXPBUFFER_H -# include "../src/interfaces/libpq/pqexpbuffer.h" - -#else -# include "../src/interfaces/libpq/pqexpbuffer.h" -#endif - +#include "../src/interfaces/libpq/pqexpbuffer.h" // option_parse_int parse_sync_method #include "../src/fe_utils/option_utils.c" diff --git a/pglite/includes/pglite-comm.h b/pglite/includes/pglite-comm.h new file mode 100644 index 0000000000000..2a84cbf406d2e --- /dev/null +++ b/pglite/includes/pglite-comm.h @@ -0,0 +1,88 @@ +#if defined(__EMSCRIPTEN__) + +#ifndef PGLITE_COMM_H +#define PGLITE_COMM_H + +#include + +volatile int querylen = 0; +volatile FILE* queryfp = NULL; + +// read FROM JS +// (i guess return number of bytes written) +// ssize_t pglite_read(/* ignored */ int socket, void *buffer, size_t length,/* ignored */ int flags,/* ignored */ void *address,/* ignored */ socklen_t *address_len); +//typedef ssize_t (*pglite_read_t)(/* ignored */ int socket, void *buffer, size_t length,/* ignored */ int flags,/* ignored */ void *address,/* ignored */ unsigned int *address_len); +typedef ssize_t (*pglite_read_t)(void *buffer, size_t max_length); +pglite_read_t pglite_read; + +// write TO JS +// (i guess return number of bytes read) +// ssize_t pglite_write(/* ignored */ int sockfd, const void *buf, size_t len, /* ignored */ int flags); +// typedef ssize_t (*pglite_write_t)(/* ignored */ int sockfd, const void *buf, size_t len, /* ignored */ int flags); +typedef ssize_t (*pglite_write_t)(void *buffer, size_t length); +pglite_write_t pglite_write; + +__attribute__((export_name("set_read_write_cbs"))) +void +set_read_write_cbs(pglite_read_t read_cb, pglite_write_t write_cb) { + pglite_read = read_cb; + pglite_write = write_cb; +} + +int EMSCRIPTEN_KEEPALIVE fcntl(int __fd, int __cmd, ...) { + // dummy + return 0; +} + +int EMSCRIPTEN_KEEPALIVE setsockopt(int __fd, int __level, int __optname, + const void *__optval, socklen_t __optlen) { + // dummy + return 0; +} + +int EMSCRIPTEN_KEEPALIVE getsockopt(int __fd, int __level, int __optname, + void *__restrict __optval, + socklen_t *__restrict __optlen) { + // dummy + return 0; +} + +int EMSCRIPTEN_KEEPALIVE getsockname(int __fd, struct sockaddr * __addr, + socklen_t *__restrict __len) { + // dummy + return 0; + } + +ssize_t EMSCRIPTEN_KEEPALIVE + recv(int __fd, void *__buf, size_t __n, int __flags) { + ssize_t got = pglite_read(__buf, __n); + return got; + } + +ssize_t EMSCRIPTEN_KEEPALIVE + send(int __fd, const void *__buf, size_t __n, int __flags) { + ssize_t wrote = pglite_write(__buf, __n); + return wrote; + } + +int EMSCRIPTEN_KEEPALIVE + connect(int socket, const struct sockaddr *address, socklen_t address_len) { + // dummy + return 0; + } + +struct pollfd { + int fd; /* file descriptor */ + short events; /* requested events */ + short revents; /* returned events */ +}; + +int EMSCRIPTEN_KEEPALIVE + poll(struct pollfd fds[], ssize_t nfds, int timeout) { + // dummy + return nfds; + } + +#endif + +#endif // ifndef PGLITE_COMM_H \ No newline at end of file diff --git a/src/backend/libpq/auth.c b/src/backend/libpq/auth.c index 4ae4ed67a3d8b..956d6f294595b 100644 --- a/src/backend/libpq/auth.c +++ b/src/backend/libpq/auth.c @@ -1856,7 +1856,7 @@ auth_peer(hbaPort *port) { uid_t uid; gid_t gid; -#if !defined(WIN32) && !defined(__wasi__) +#if !defined(WIN32) struct passwd *pw; int ret; #endif @@ -1875,7 +1875,7 @@ auth_peer(hbaPort *port) return STATUS_ERROR; } -#if !defined(WIN32) && !defined(__wasi__) +#if !defined(WIN32) errno = 0; /* clear errno before call */ pw = getpwuid(uid); if (!pw) diff --git a/src/backend/libpq/be-fsstubs.c b/src/backend/libpq/be-fsstubs.c index afc579fd20683..27d317dfdc087 100644 --- a/src/backend/libpq/be-fsstubs.c +++ b/src/backend/libpq/be-fsstubs.c @@ -150,12 +150,8 @@ be_lo_close(PG_FUNCTION_ARGS) * *****************************************************************************/ -#if defined(__EMSCRIPTEN__) || defined(__wasi__) -static int -#else int -#endif -lo_read3(int fd, char *buf, int len) +lo_read(int fd, char *buf, int len) { int status; LargeObjectDesc *lobj; @@ -182,12 +178,8 @@ lo_read3(int fd, char *buf, int len) return status; } -#if defined(__EMSCRIPTEN__) || defined(__wasi__) -static int -#else int -#endif -lo_write3(int fd, const char *buf, int len) +lo_write(int fd, const char *buf, int len) { int status; LargeObjectDesc *lobj; @@ -198,7 +190,7 @@ lo_write3(int fd, const char *buf, int len) errmsg("invalid large-object descriptor: %d", fd))); lobj = cookies[fd]; - /* see comment in lo_read3() */ + /* see comment in lo_read() */ if ((lobj->flags & IFS_WRLOCK) == 0) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), @@ -373,7 +365,7 @@ be_loread(PG_FUNCTION_ARGS) len = 0; retval = (bytea *) palloc(VARHDRSZ + len); - totalread = lo_read3(fd, VARDATA(retval), len); + totalread = lo_read(fd, VARDATA(retval), len); SET_VARSIZE(retval, totalread + VARHDRSZ); PG_RETURN_BYTEA_P(retval); @@ -390,7 +382,7 @@ be_lowrite(PG_FUNCTION_ARGS) PreventCommandIfReadOnly("lowrite()"); bytestowrite = VARSIZE_ANY_EXHDR(wbuf); - totalwritten = lo_write3(fd, VARDATA_ANY(wbuf), bytestowrite); + totalwritten = lo_write(fd, VARDATA_ANY(wbuf), bytestowrite); PG_RETURN_INT32(totalwritten); } @@ -568,7 +560,7 @@ lo_truncate_internal(int32 fd, int64 len) errmsg("invalid large-object descriptor: %d", fd))); lobj = cookies[fd]; - /* see comment in lo_read3() */ + /* see comment in lo_read() */ if ((lobj->flags & IFS_WRLOCK) == 0) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), diff --git a/src/backend/libpq/be-secure.c b/src/backend/libpq/be-secure.c index 1663f36b6b80d..b2e9260e8b1a6 100644 --- a/src/backend/libpq/be-secure.c +++ b/src/backend/libpq/be-secure.c @@ -32,6 +32,10 @@ #include "tcop/tcopprot.h" #include "utils/wait_event.h" +#if defined(__EMSCRIPTEN__) +#include "pglite-comm.h" +#endif + char *ssl_library; char *ssl_cert_file; char *ssl_key_file; diff --git a/src/backend/libpq/pqcomm.c b/src/backend/libpq/pqcomm.c index 856e9d2a3623c..e022fa25082c3 100644 --- a/src/backend/libpq/pqcomm.c +++ b/src/backend/libpq/pqcomm.c @@ -80,10 +80,6 @@ #include "utils/guc_hooks.h" #include "utils/memutils.h" -#if defined(__EMSCRIPTEN__) -#include -#endif - /* * Cope with the various platform-specific ways to spell TCP keepalive socket * options. This doesn't cover Windows, which as usual does its own thing. @@ -129,53 +125,6 @@ static size_t PqSendStart; /* Next index to send a byte in PqSendBuffer */ static char PqRecvBuffer[PQ_RECV_BUFFER_SIZE]; static int PqRecvPointer; /* Next index to read a byte from PqRecvBuffer */ static int PqRecvLength; /* End of data available in PqRecvBuffer */ -#if defined(__EMSCRIPTEN__) || defined(__wasi__) -volatile int querylen = 0; -volatile FILE* queryfp = NULL; - -typedef ssize_t (*pglite_read_t)(void *buffer, size_t max_length); -extern pglite_read_t pglite_read; - -typedef ssize_t(*pglite_write_t)(void *buffer, size_t length); -extern pglite_write_t pglite_write; - -int EMSCRIPTEN_KEEPALIVE fcntl(int __fd, int __cmd, ...) { - // dummy - return 0; -} - -int EMSCRIPTEN_KEEPALIVE setsockopt(int __fd, int __level, int __optname, - const void *__optval, socklen_t __optlen) { - // dummy - return 0; -} - -int EMSCRIPTEN_KEEPALIVE getsockopt(int __fd, int __level, int __optname, - void *__restrict __optval, - socklen_t *__restrict __optlen) { - // dummy - return 0; -} - -int EMSCRIPTEN_KEEPALIVE getsockname(int __fd, struct sockaddr * __addr, - socklen_t *__restrict __len) { - // dummy - return 0; - } - -ssize_t EMSCRIPTEN_KEEPALIVE - recv(int __fd, void *__buf, size_t __n, int __flags) { - ssize_t got = pglite_read(__buf, __n); - return got; - } - -ssize_t EMSCRIPTEN_KEEPALIVE - send(int __fd, const void *__buf, size_t __n, int __flags) { - ssize_t wrote = pglite_write(__buf, __n); - return wrote; - } - -#endif /* * Message status diff --git a/src/backend/storage/ipc/latch.c b/src/backend/storage/ipc/latch.c index aeeab4544693f..2b4ded38893af 100644 --- a/src/backend/storage/ipc/latch.c +++ b/src/backend/storage/ipc/latch.c @@ -231,7 +231,6 @@ ResourceOwnerForgetWaitEventSet(ResourceOwner owner, WaitEventSet *set) void InitializeLatchSupport(void) { -#if !defined(__wasi__) #if defined(WAIT_USE_SELF_PIPE) int pipefd[2]; @@ -341,7 +340,6 @@ InitializeLatchSupport(void) /* Ignore SIGURG, because we'll receive it via kqueue. */ pqsignal(SIGURG, SIG_IGN); #endif -#endif /* __wasi__ */ } void diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index bbcf4112aead6..a9bfd973eb12e 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -109,12 +109,6 @@ int restrict_nonsystem_relation_kind; #if (defined(__EMSCRIPTEN__) || defined(__wasi__)) bool quote_all_identifiers = false; - -typedef ssize_t (*pglite_read_t)(void *buffer, size_t max_length); -pglite_read_t pglite_read = NULL; - -typedef ssize_t(*pglite_write_t)(void *buffer, size_t length); -pglite_write_t pglite_write = NULL; #endif // WASM @@ -5146,9 +5140,7 @@ ShowUsage(const char *title) (long) user.tv_usec, (long) sys.tv_sec, (long) sys.tv_usec); -#if !defined(__wasi__) #ifndef WIN32 -#if !defined(__wasi__) /* * The following rusage fields are not defined by POSIX, but they're * present on all current Unix-like systems so we use them without any @@ -5190,9 +5182,7 @@ ShowUsage(const char *title) r.ru_nvcsw - Save_r.ru_nvcsw, r.ru_nivcsw - Save_r.ru_nivcsw, r.ru_nvcsw, r.ru_nivcsw); -#endif /* !__wasi__ */ #endif /* !WIN32 */ -#endif /* remove trailing newline */ if (str.data[str.len - 1] == '\n') str.data[--str.len] = '\0'; diff --git a/src/backend/utils/error/elog.c b/src/backend/utils/error/elog.c index f8dc8562571a7..dcd1ea8f050ed 100644 --- a/src/backend/utils/error/elog.c +++ b/src/backend/utils/error/elog.c @@ -543,12 +543,7 @@ errfinish(const char *filename, int lineno, const char *funcname) */ recursion_depth--; -#if defined(__wasi__) - fprintf(stderr, "# 547: PG_RE_THROW(ERROR : %d) ignored\r\n", recursion_depth); - abort(); -#else PG_RE_THROW(); -#endif } /* Emit the message to the right places */ diff --git a/src/backend/utils/fmgr/dfmgr.c b/src/backend/utils/fmgr/dfmgr.c index 55c606dc74631..092004dcf3b3f 100644 --- a/src/backend/utils/fmgr/dfmgr.c +++ b/src/backend/utils/fmgr/dfmgr.c @@ -121,15 +121,12 @@ load_external_function(const char *filename, const char *funcname, /* Look up the function within the library. */ retval = dlsym(lib_handle, funcname); -#if !defined(__wasi__) + if (retval == NULL && signalNotFound) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_FUNCTION), errmsg("could not find function \"%s\" in file \"%s\"", funcname, fullname))); -#else - fprintf(stderr, "could not find function \"%s\" in file \"%s\" rv=%p snf=%b\n", funcname, fullname, retval, signalNotFound); -#endif pfree(fullname); return retval; diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 97c44df762fd6..ea076bf59903a 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -2647,6 +2647,7 @@ ReportGUCOption(struct config_generic *record) pq_sendstring(&msgbuf, val); pq_endmessage(&msgbuf); +#ifndef __EMSCRIPTEN__ /* * We need a long-lifespan copy. If guc_strdup() fails due to OOM, * we'll set last_reported to NULL and thereby possibly make a @@ -2654,6 +2655,7 @@ ReportGUCOption(struct config_generic *record) */ guc_free(record->last_reported); record->last_reported = guc_strdup(LOG, val); +#endif } pfree(val); diff --git a/src/bin/pg_config/Makefile b/src/bin/pg_config/Makefile index 3d3c44fa2b4b8..41c0119270d9d 100644 --- a/src/bin/pg_config/Makefile +++ b/src/bin/pg_config/Makefile @@ -21,7 +21,7 @@ OBJS = \ all: ifeq ($(PORTNAME), emscripten) -all: pg_config pg_config_sh +all: pg_config else all: pg_config endif @@ -31,18 +31,16 @@ pg_config: $(OBJS) | submake-libpgport # the emscripten build generates multiple "runnable" files (.cjs/.mjs + .wasm) # but other postgres tools expect exacly 'pg_config' as an executable -# the following rule creates a node script that calls the pg_config.cjs/.mjs +# we create a pg_config link to run_pg_config.mjs, which in turn calls pg_config.cjs/.mjs # pg_config.wasm is implicitly expected to be in the same folder -pg_config_sh: pg_config - echo "#!/usr/bin/env node" > pg_config - echo 'require("./pg_config$(X)")' >> pg_config - chmod +x pg_config install: all installdirs $(INSTALL_SCRIPT) pg_config$(X) '$(DESTDIR)$(bindir)/pg_config$(X)' ifeq ($(PORTNAME), emscripten) $(INSTALL_PROGRAM) pg_config.wasm '$(DESTDIR)$(bindir)/pg_config.wasm' - $(INSTALL_PROGRAM) pg_config '$(DESTDIR)$(bindir)/pg_config' + $(INSTALL_PROGRAM) pg_config$(X) '$(DESTDIR)$(bindir)/pg_config.mjs' + $(INSTALL_PROGRAM) run_pg_config.mjs '$(DESTDIR)$(bindir)/run_pg_config.mjs' + ln -sf '$(DESTDIR)$(bindir)/run_pg_config.mjs' '$(DESTDIR)$(bindir)/pg_config' endif installdirs: diff --git a/src/bin/pg_config/run_pg_config.mjs b/src/bin/pg_config/run_pg_config.mjs new file mode 100644 index 0000000000000..ae1dd000b1be3 --- /dev/null +++ b/src/bin/pg_config/run_pg_config.mjs @@ -0,0 +1,12 @@ +#!/usr/bin/env node +import Module from "./pg_config.mjs"; + +// If the module has a `main()` (or similar) method, call it: +if (typeof Module.main === "function") { + await Module.main(); +} else if (typeof Module === "function") { + // If it's a callable function/class, instantiate or run it + await Module(); +} else { + console.error("⚠️ Module has no callable entry point"); +} diff --git a/src/bin/pg_dump/parallel.c b/src/bin/pg_dump/parallel.c index c1b141a5c32cb..a09247fae47e3 100644 --- a/src/bin/pg_dump/parallel.c +++ b/src/bin/pg_dump/parallel.c @@ -49,7 +49,7 @@ * The pstate->te[] entry for each worker is valid when it's in WRKR_WORKING * state, and must be NULL in other states. */ -#define PG_DUMP_PARALLEL + #include "postgres_fe.h" #ifndef WIN32 @@ -445,7 +445,6 @@ ShutdownWorkersHard(ParallelState *pstate) static void WaitForTerminatingWorkers(ParallelState *pstate) { -#if !defined(__wasi__) while (!HasEveryWorkerTerminated(pstate)) { ParallelSlot *slot = NULL; @@ -505,7 +504,6 @@ WaitForTerminatingWorkers(ParallelState *pstate) slot->workerStatus = WRKR_TERMINATED; pstate->te[j] = NULL; } -#endif /* __wasi__ */ } diff --git a/src/bin/pg_dump/pg_backup_db.c b/src/bin/pg_dump/pg_backup_db.c index f57d971a23c3f..ff89ae75aee4b 100644 --- a/src/bin/pg_dump/pg_backup_db.c +++ b/src/bin/pg_dump/pg_backup_db.c @@ -292,7 +292,7 @@ ExecuteSqlQuery(Archive *AHX, const char *query, ExecStatusType status) ArchiveHandle *AH = (ArchiveHandle *) AHX; PGresult *res; - res = PQexec(AH->connection, query); printf("# 295:" __FILE__ ": PQexec fail == %d q=%s\r\n", PQresultStatus(res) != status, query ); + res = PQexec(AH->connection, query); if (PQresultStatus(res) != status) die_on_query_failure(AH, query); return res; diff --git a/src/bin/pg_dump/pg_dump.c b/src/bin/pg_dump/pg_dump.c index acfa76da4e89b..22699405a2ef6 100644 --- a/src/bin/pg_dump/pg_dump.c +++ b/src/bin/pg_dump/pg_dump.c @@ -69,28 +69,8 @@ #include "pg_dump.h" #include "storage/block.h" -#if !defined(__wasi__) && !defined(__EMSCRIPTEN__) -extern PGDLLIMPORT bool fe_utils_quote_all_identifiers; -#else -#if defined(WASM) -#ifdef errno - #undef errno - int errno; -#endif - -pid_t fork(void) { - return -1; -} - -# include - -__attribute__((export_name("sdk_fd_seek"))) -int sdk_fd_seek(int fd, int offset,int whence, unsigned long long *retptr) { - puts("sdk_fd_seek called !"); - return __wasi_fd_seek(fd, offset, whence, retptr); -} - -#endif +#if defined(__EMSCRIPTEN__) +#include #endif typedef struct @@ -366,8 +346,10 @@ static TableInfo *getRootTableInfo(const TableInfo *tbinfo); static bool forcePartitionRootLoad(const TableInfo *tbinfo); static void read_dump_filters(const char *filename, DumpOptions *dopt); - -int +int +#if defined(__EMSCRIPTEN__) +EMSCRIPTEN_KEEPALIVE +#endif main(int argc, char **argv) { int c; @@ -853,13 +835,11 @@ main(int argc, char **argv) fout->maxRemoteVersion = (PG_VERSION_NUM / 100) * 100 + 99; fout->numWorkers = numWorkers; -puts("# 832:" __FILE__ ": ConnectDatabase\r\n"); /* * Open the database using the Archiver, so it knows about it. Errors mean * death. */ ConnectDatabase(fout, &dopt.cparams, false); -puts("# 838:" __FILE__ ": ConnectDatabase->setup_connection\r\n"); setup_connection(fout, dumpencoding, dumpsnapshot, use_role); /* diff --git a/src/include/libpq/be-fsstubs.h b/src/include/libpq/be-fsstubs.h index 0d6c7124b5400..336b9cef12508 100644 --- a/src/include/libpq/be-fsstubs.h +++ b/src/include/libpq/be-fsstubs.h @@ -19,10 +19,8 @@ * Probably these should have had the underscore-free names, * but too late now... */ -#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) -extern int lo_read3(int fd, char *buf, int len); -extern int lo_write3(int fd, const char *buf, int len); -#endif +extern int lo_read(int fd, char *buf, int len); +extern int lo_write(int fd, const char *buf, int len); /* * Cleanup LOs at xact commit/abort diff --git a/src/interfaces/libpq/fe-connect.c b/src/interfaces/libpq/fe-connect.c index f8138e3d490cd..4c81641d3904f 100644 --- a/src/interfaces/libpq/fe-connect.c +++ b/src/interfaces/libpq/fe-connect.c @@ -2033,7 +2033,6 @@ PQsetdbLogin(const char *pghost, const char *pgport, const char *pgoptions, static int connectNoDelay(PGconn *conn) { -#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) #ifdef TCP_NODELAY int on = 1; @@ -2047,7 +2046,6 @@ connectNoDelay(PGconn *conn) SOCK_STRERROR(SOCK_ERRNO, sebuf, sizeof(sebuf))); return 0; } -#endif #endif return 1; } @@ -2169,9 +2167,6 @@ connectFailureMessage(PGconn *conn, int errorno) static int useKeepalives(PGconn *conn) { -#if defined(__EMSCRIPTEN__) || defined(__wasi__) -return 0; -#else int val; if (conn->keepalives == NULL) @@ -2181,7 +2176,6 @@ return 0; return -1; return val != 0 ? 1 : 0; -#endif } #ifndef WIN32 @@ -2408,14 +2402,12 @@ pqConnectDBStart(PGconn *conn) * Nobody but developers should see this message, so we don't bother * translating it. */ -#if !defined(__EMSCRIPTEN__) && !defined(__wasi__) if (!pg_link_canary_is_frontend()) { appendPQExpBufferStr(&conn->errorMessage, "libpq is incorrectly linked to backend functions\n"); goto connect_errReturn; } -#endif /* Ensure our buffers are empty */ conn->inStart = conn->inCursor = conn->inEnd = 0; conn->outCount = 0; @@ -2501,7 +2493,6 @@ pqConnectDBComplete(PGconn *conn) for (;;) { int ret = 0; -#if !defined(__wasi__) /* * (Re)start the connect_timeout timer if it's active and we are * considering a different host than we were last time through. If @@ -2516,8 +2507,6 @@ pqConnectDBComplete(PGconn *conn) last_whichhost = conn->whichhost; last_whichaddr = conn->whichaddr; } -#endif -printf("# 2519: switch (%d) PGRES_POLLING_OK=%d PGRES_POLLING_READING=%d PGRES_POLLING_WRITING=%d\n", flag, PGRES_POLLING_OK, PGRES_POLLING_READING,PGRES_POLLING_WRITING); if(!flag) abort(); /* * Wait, if necessary. Note that the initial state (just after @@ -2529,7 +2518,6 @@ if(!flag) abort(); return 1; /* success! */ case PGRES_POLLING_READING: -#if !defined(__wasi__) ret = pqWaitTimed(1, 0, conn, end_time); if (ret == -1) { @@ -2537,11 +2525,9 @@ if(!flag) abort(); conn->status = CONNECTION_BAD; return 0; } -#endif break; case PGRES_POLLING_WRITING: -#if !defined(__wasi__) ret = pqWaitTimed(0, 1, conn, end_time); if (ret == -1) { @@ -2549,7 +2535,6 @@ if(!flag) abort(); conn->status = CONNECTION_BAD; return 0; } -#endif break; default: @@ -2637,14 +2622,7 @@ PQconnectPoll(PGconn *conn) case CONNECTION_CHECK_STANDBY: { /* Load waiting data */ -#if defined(__wasi__) - int n = pqReadData(conn); - if (!n) { - sched_yield(); - } -#else -int n = pqReadData(conn); -#endif + int n = pqReadData(conn); if (n < 0) goto error_return; if (n == 0) @@ -3238,7 +3216,6 @@ int n = pqReadData(conn); * Now check (using getsockopt) that there is not an error * state waiting for us on the socket. */ -#if !defined(__wasi__) if (getsockopt(conn->sock, SOL_SOCKET, SO_ERROR, (char *) &optval, &optlen) == -1) { @@ -3325,10 +3302,6 @@ int n = pqReadData(conn); /* * Make sure we can write before advancing to next step. */ -#else -conn->options_valid = true; -conn->try_next_host = false; -#endif // __wasi__ conn->status = CONNECTION_MADE; return PGRES_POLLING_WRITING; } @@ -3895,12 +3868,6 @@ conn->try_next_host = false; * Note that conn->pghost must be non-NULL if we are going to * avoid the Kerberos code doing a hostname look-up. */ -#if defined(__wasi__) -if (!conn->pghost) { - conn->pgpass = strdup("md532e12f215ba27cb750c9e093ce4b5127"); - conn->pghost = strdup("localhost"); -} -#endif res = pg_fe_sendauth(areq, msgLength, conn); /* OK, we have processed the message; mark data consumed */ diff --git a/src/interfaces/libpq/fe-secure.c b/src/interfaces/libpq/fe-secure.c index f628082337ebc..7f170f55730db 100644 --- a/src/interfaces/libpq/fe-secure.c +++ b/src/interfaces/libpq/fe-secure.c @@ -45,6 +45,10 @@ #include "libpq-fe.h" #include "libpq-int.h" +#if defined(__EMSCRIPTEN__) +#include "pglite-comm.h" +#endif + /* * Macros to handle disabling and then restoring the state of SIGPIPE handling. * On Windows, these are all no-ops since there's no SIGPIPEs. diff --git a/src/port/pqsignal.c b/src/port/pqsignal.c index fa9317bca314d..53cec67123c23 100644 --- a/src/port/pqsignal.c +++ b/src/port/pqsignal.c @@ -174,13 +174,4 @@ pqsignal(int signo, pqsigfunc func) #endif } -/* sneak emsdk or wasi wasm port support into libpgport */ -#if defined(SDK_PORT) -# if defined(__wasi__) -# include "sdk_port-wasi.c" -# endif -# if defined(__EMSCRIPTEN__) -// # include "sdk_port-emscripten.c" -# endif -#endif From 73abc36304b04ce27f0a5f70582959c40f876b5e Mon Sep 17 00:00:00 2001 From: Tudor Zaharia <37596026+tdrz@users.noreply.github.com> Date: Wed, 5 Nov 2025 19:00:01 +0100 Subject: [PATCH 11/11] Added backend implementation for pg_uuidv7 extension (#53) * backend pg_uuidv7 implemented --------- Co-authored-by: tudor --- .gitmodules | 3 +++ pglite/Makefile | 3 ++- pglite/pg_uuidv7 | 1 + src/backend/Makefile | 4 ++-- src/include/port/pg_pthread.h | 3 --- 5 files changed, 8 insertions(+), 6 deletions(-) create mode 160000 pglite/pg_uuidv7 diff --git a/.gitmodules b/.gitmodules index 94158709d174c..60cfbe40928c7 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "pglite/pgtap"] path = pglite/pgtap url = https://github.com/theory/pgtap.git +[submodule "pglite/pg_uuidv7"] + path = pglite/pg_uuidv7 + url = https://github.com/fboulnois/pg_uuidv7/ diff --git a/pglite/Makefile b/pglite/Makefile index f6bc99b8e50cc..633fe686b9eba 100644 --- a/pglite/Makefile +++ b/pglite/Makefile @@ -6,7 +6,8 @@ include $(top_builddir)/src/Makefile.global SUBDIRS = \ pg_ivm \ vector \ - pgtap + pgtap \ + pg_uuidv7 prefix ?= /install/pglite EXTENSIONS_BUILD_ROOT := /tmp/extensions/build diff --git a/pglite/pg_uuidv7 b/pglite/pg_uuidv7 new file mode 160000 index 0000000000000..c707aae241118 --- /dev/null +++ b/pglite/pg_uuidv7 @@ -0,0 +1 @@ +Subproject commit c707aae2411181be4802f5fa565b44d9c0bcbc29 diff --git a/src/backend/Makefile b/src/backend/Makefile index 0bc6a84c3b10a..1c04efeab0099 100644 --- a/src/backend/Makefile +++ b/src/backend/Makefile @@ -110,8 +110,8 @@ pglite: pglite-exported-functions $(CC) $(CFLAGS) $(LDFLAGS) -DPG_PREFIX=/tmp/pglite -I$(top_builddir)/src/include -I$(top_builddir)/src/ -I$(top_builddir)/src/interfaces/libpq -o pglite.o -c $(top_builddir)/$(PGLITE_MAIN) -Wno-incompatible-pointer-types-discards-qualifiers $(CC) \ $(PGLITE_CFLAGS) \ - -fPIC -m32 -D_FILE_OFFSET_BITS=64 -mno-bulk-memory -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-extended-const -mno-atomics -mno-tail-call -mno-multivalue -mno-relaxed-simd -mno-simd128 -mno-multimemory -mno-exception-handling -Wno-unused-command-line-argument -Wno-unreachable-code-fallthrough -Wno-unused-function -Wno-invalid-noreturn -Wno-declaration-after-statement -Wno-invalid-noreturn \ - -DPYDK=1 -DPG_PREFIX=/tmp/pglite -o pglite.html \ + -fPIC -m32 -mno-bulk-memory -mnontrapping-fptoint -mno-reference-types -mno-sign-ext -mno-extended-const -mno-atomics -mno-tail-call -mno-multivalue -mno-relaxed-simd -mno-simd128 -mno-multimemory -mno-exception-handling -Wno-unused-command-line-argument -Wno-unreachable-code-fallthrough -Wno-unused-function -Wno-invalid-noreturn -Wno-declaration-after-statement -Wno-invalid-noreturn \ + -DPG_PREFIX=/tmp/pglite -o pglite.html \ $(PGPRELOAD) \ -ferror-limit=1 \ -sEXPORTED_FUNCTIONS=@$(emscripten_imports_dir)/exported_functions.txt \ diff --git a/src/include/port/pg_pthread.h b/src/include/port/pg_pthread.h index 6922eb423b945..b3d4ffbf59353 100644 --- a/src/include/port/pg_pthread.h +++ b/src/include/port/pg_pthread.h @@ -12,9 +12,6 @@ #ifndef PG_PTHREAD_H #define PG_PTHREAD_H -#if defined(__wasi__) -#define PYDK -#endif /* __wasi__ */ #include #ifndef HAVE_PTHREAD_BARRIER_WAIT