#include "lmdb-js.h" #include using namespace Napi; void setFlagFromValue(int *flags, int flag, const char *name, bool defaultValue, Object options); DbiWrap::DbiWrap(const Napi::CallbackInfo& info) : ObjectWrap(info) { this->dbi = 0; this->keyType = LmdbKeyType::DefaultKey; this->compression = nullptr; this->isOpen = false; this->getFast = false; this->ew = nullptr; EnvWrap *ew; napi_unwrap(info.Env(), info[0], (void**) &ew); this->env = ew->env; this->ew = ew; int flags = info[1].As(); char* nameBytes; std::string name; if (info[2].IsString()) { name = info[2].As().Utf8Value(); nameBytes = (char*) name.c_str(); } else nameBytes = nullptr; LmdbKeyType keyType = (LmdbKeyType) info[3].As().Int32Value(); Compression* compression; if (info[4].IsObject()) napi_unwrap(info.Env(), info[4], (void**) &compression); else compression = nullptr; int rc = this->open(flags, nameBytes, flags & HAS_VERSIONS, keyType, compression); //if (nameBytes) //delete nameBytes; if (rc) { if (rc == MDB_NOTFOUND) this->dbi = (MDB_dbi) 0xffffffff; else { //delete this; throwLmdbError(info.Env(), rc); return; } } info.This().As().Set("dbi", Number::New(info.Env(), this->dbi)); info.This().As().Set("address", Number::New(info.Env(), (size_t) this)); } DbiWrap::~DbiWrap() { // Imagine the following JS: // ------------------------ // var dbi1 = env.openDbi({ name: "hello" }); // var dbi2 = env.openDbi({ name: "hello" }); // dbi1.close(); // txn.putString(dbi2, "world"); // ----- // The above DbiWrap objects would both wrap the same MDB_dbi, and if closing the first one called mdb_dbi_close, // that'd also render the second DbiWrap instance unusable. // // For this reason, we will never call mdb_dbi_close // NOTE: according to LMDB authors, it is perfectly fine if mdb_dbi_close is never called on an MDB_dbi } int DbiWrap::open(int flags, char* name, bool hasVersions, LmdbKeyType keyType, Compression* compression) { MDB_txn* txn = ew->getReadTxn(); this->hasVersions = hasVersions; this->compression = compression; this->keyType = keyType; #ifndef MDB_RPAGE_CACHE flags &= ~0x100; // remove use versions flag for older lmdb #endif this->flags = flags; int rc = txn ? mdb_dbi_open(txn, name, flags, &this->dbi) : EINVAL; if (rc) return rc; this->isOpen = true; if (keyType == LmdbKeyType::DefaultKey && name) { // use the fast compare, but can't do it if we have db table/names mixed in mdb_set_compare(txn, dbi, compareFast); } return 0; } Value DbiWrap::close(const Napi::CallbackInfo& info) { if (this->isOpen) { mdb_dbi_close(this->env, this->dbi); this->isOpen = false; this->ew = nullptr; } else { return throwError(info.Env(), "The Dbi is not open, you can't close it."); } return info.Env().Undefined(); } Value DbiWrap::drop(const Napi::CallbackInfo& info) { int del = 1; int rc; if (!this->isOpen) { return throwError(info.Env(), "The Dbi is not open, you can't drop it."); } // Check if the database should be deleted if (info.Length() == 1 && info[0].IsObject()) { Napi::Object options = info[0].As(); // Just free pages Napi::Value opt = options.Get("justFreePages"); del = opt.IsBoolean() ? !opt.As().Value() : 1; } // Drop database rc = mdb_drop(ew->writeTxn->txn, dbi, del); if (rc != 0) { return throwLmdbError(info.Env(), rc); } // Only close database if del == 1 if (del == 1) { isOpen = false; ew = nullptr; } return info.Env().Undefined(); } Value DbiWrap::stat(const Napi::CallbackInfo& info) { MDB_stat stat; mdb_stat(this->ew->getReadTxn(), dbi, &stat); Object stats = Object::New(info.Env()); stats.Set("pageSize", Number::New(info.Env(), stat.ms_psize)); stats.Set("treeDepth", Number::New(info.Env(), stat.ms_depth)); stats.Set("treeBranchPageCount", Number::New(info.Env(), stat.ms_branch_pages)); stats.Set("treeLeafPageCount", Number::New(info.Env(), stat.ms_leaf_pages)); stats.Set("entryCount", Number::New(info.Env(), stat.ms_entries)); stats.Set("overflowPages", Number::New(info.Env(), stat.ms_overflow_pages)); return stats; } int32_t DbiWrap::doGetByBinary(uint32_t keySize, uint32_t ifNotTxnId, int64_t txnWrapAddress) { char* keyBuffer = ew->keyBuffer; MDB_txn* txn = ew->getReadTxn(txnWrapAddress); MDB_val key, data; key.mv_size = keySize; key.mv_data = (void*) keyBuffer; uint32_t* currentTxnId = (uint32_t*) (keyBuffer + 32); #ifdef MDB_RPAGE_CACHE int result = mdb_get_with_txn(txn, dbi, &key, &data, (mdb_size_t*) currentTxnId); #else int result = mdb_get(txn, dbi, &key, &data); #endif if (result) { if (result > 0) return -result; return result; } #ifdef MDB_RPAGE_CACHE if (ifNotTxnId && ifNotTxnId == *currentTxnId) return -30004; #endif result = getVersionAndUncompress(data, this); bool fits = true; if (result) { fits = valToBinaryFast(data, this); // it fits in the global/compression-target buffer } #if ENABLE_V8_API // TODO: We may want to enable this for Bun as well, since it probably doesn't have the same // shared pointer problems that V8 does if (fits || result == 2 || data.mv_size < SHARED_BUFFER_THRESHOLD) {// result = 2 if it was decompressed #endif if (data.mv_size < 0x80000000) return data.mv_size; *((uint32_t*)keyBuffer) = data.mv_size; return -30000; #if ENABLE_V8_API } else { return EnvWrap::toSharedBuffer(ew->env, (uint32_t*) ew->keyBuffer, data); } #endif } NAPI_FUNCTION(directWrite) { ARGS(5) GET_INT64_ARG(0); DbiWrap* dw = (DbiWrap*) i64; uint32_t keySize; GET_UINT32_ARG(keySize, 1); uint32_t offset; GET_UINT32_ARG(offset, 2); uint32_t dataSize; GET_UINT32_ARG(dataSize, 3); int64_t txnAddress = 0; napi_status status = napi_get_value_int64(env, args[4], &txnAddress); if (dw->hasVersions) offset += 8; EnvWrap* ew = dw->ew; char* keyBuffer = ew->keyBuffer; MDB_txn* txn = ew->getReadTxn(txnAddress); MDB_val key, data; key.mv_size = keySize; key.mv_data = (void*) keyBuffer; data.mv_size = dataSize; data.mv_data = (void*) (keyBuffer + (((keySize >> 3) + 1) << 3)); #ifdef MDB_RPAGE_CACHE int result = mdb_direct_write(txn, dw->dbi, &key, offset, &data); #else int result = -1; #endif RETURN_INT32(result); } NAPI_FUNCTION(getByBinary) { ARGS(4) GET_INT64_ARG(0); DbiWrap* dw = (DbiWrap*) i64; uint32_t keySize; GET_UINT32_ARG(keySize, 1); uint32_t ifNotTxnId; GET_UINT32_ARG(ifNotTxnId, 2); int64_t txnAddress = 0; napi_status status = napi_get_value_int64(env, args[3], &txnAddress); RETURN_INT32(dw->doGetByBinary(keySize, ifNotTxnId, txnAddress)); } uint32_t getByBinaryFFI(double dwPointer, uint32_t keySize, uint32_t ifNotTxnId, uint64_t txnAddress) { DbiWrap* dw = (DbiWrap*) (size_t) dwPointer; return dw->doGetByBinary(keySize, ifNotTxnId, txnAddress); } napi_finalize noopDbi = [](napi_env, void *, void *) { // Data belongs to LMDB, we shouldn't free it here }; NAPI_FUNCTION(getSharedByBinary) { ARGS(2) GET_INT64_ARG(0); DbiWrap* dw = (DbiWrap*) i64; uint32_t keySize; GET_UINT32_ARG(keySize, 1); MDB_val key; MDB_val data; key.mv_size = keySize; key.mv_data = (void*) dw->ew->keyBuffer; MDB_txn* txn = dw->ew->getReadTxn(); int rc = mdb_get(txn, dw->dbi, &key, &data); if (rc) { if (rc == MDB_NOTFOUND) { RETURN_UNDEFINED; } else return throwLmdbError(env, rc); } rc = getVersionAndUncompress(data, dw); napi_create_external_buffer(env, data.mv_size, (char*) data.mv_data, noopDbi, nullptr, &returnValue); return returnValue; } NAPI_FUNCTION(getStringByBinary) { ARGS(3) GET_INT64_ARG(0); DbiWrap* dw = (DbiWrap*) i64; uint32_t keySize; GET_UINT32_ARG(keySize, 1); int64_t txnAddress = 0; napi_status status = napi_get_value_int64(env, args[2], &txnAddress); MDB_val key; MDB_val data; key.mv_size = keySize; key.mv_data = (void*) dw->ew->keyBuffer; MDB_txn* txn = dw->ew->getReadTxn(txnAddress); int rc = mdb_get(txn, dw->dbi, &key, &data); if (rc) { if (rc == MDB_NOTFOUND) { RETURN_UNDEFINED; } else return throwLmdbError(env, rc); } rc = getVersionAndUncompress(data, dw); if (rc) napi_create_string_utf8(env, (char*) data.mv_data, data.mv_size, &returnValue); else napi_create_int32(env, data.mv_size, &returnValue); return returnValue; } int DbiWrap::prefetch(uint32_t* keys) { MDB_txn* txn = ExtendedEnv::getPrefetchReadTxn(ew->env); MDB_val key; MDB_val data; unsigned int flags; mdb_dbi_flags(txn, dbi, &flags); bool findAllValues = flags & MDB_DUPSORT; int effected = 0; bool findDataValue = false; MDB_cursor *cursor; int rc = mdb_cursor_open(txn, dbi, &cursor); if (rc) { ExtendedEnv::donePrefetchReadTxn(txn); return rc; } while((key.mv_size = *keys++) > 0) { if (key.mv_size == 0xffffffff) { // it is a pointer to a new buffer keys = (uint32_t*) (size_t) *((double*) keys); // read as a double pointer key.mv_size = *keys++; if (key.mv_size == 0) break; } if (key.mv_size & 0x80000000) { // indicator of using a data value combination (with dupSort), to be followed by the corresponding key data.mv_size = key.mv_size & 0x7fffffff; data.mv_data = (void *) keys; keys += (data.mv_size + 12) >> 2; findDataValue = true; findAllValues = false; continue; } // else standard key key.mv_data = (void *) keys; keys += (key.mv_size + 12) >> 2; int rc = mdb_cursor_get(cursor, &key, &data, findDataValue ? MDB_GET_BOTH : MDB_SET_KEY); findDataValue = false; while (!rc) { // access one byte from each of the pages to ensure they are in the OS cache, // potentially triggering the hard page fault in this thread int pages = (data.mv_size + 0xfff) >> 12; // TODO: Adjust this for the page headers, I believe that makes the first page slightly less 4KB. for (int i = 0; i < pages; i++) { effected += *(((uint8_t*)data.mv_data) + (i << 12)); } if (findAllValues) // in dupsort databases, access the rest of the values rc = mdb_cursor_get(cursor, &key, &data, MDB_NEXT_DUP); else rc = 1; // done } } mdb_cursor_close(cursor); ExtendedEnv::donePrefetchReadTxn(txn); return effected; } class PrefetchWorker : public AsyncWorker { public: PrefetchWorker(DbiWrap* dw, uint32_t* keys, const Function& callback) : AsyncWorker(callback), dw(dw), keys(keys) {} void Execute() { dw->prefetch(keys); } void OnOK() { napi_value result; // we use direct napi call here because node-addon-api interface with throw a fatal error if a worker thread is terminating napi_call_function(Env(), Env().Undefined(), Callback().Value(), 0, {}, &result); } void OnError(const Error& e) { napi_value result; // we use direct napi call here because node-addon-api interface with throw a fatal error if a worker thread is terminating napi_value arg = e.Value(); napi_call_function(Env(), Env().Undefined(), Callback().Value(), 1, &arg, &result); } private: DbiWrap* dw; uint32_t* keys; }; NAPI_FUNCTION(prefetchNapi) { ARGS(3) GET_INT64_ARG(0); DbiWrap* dw = (DbiWrap*) i64; napi_get_value_int64(env, args[1], &i64); uint32_t* keys = (uint32_t*) i64; PrefetchWorker* worker = new PrefetchWorker(dw, keys, Function(env, args[2])); worker->Queue(); RETURN_UNDEFINED; } void DbiWrap::setupExports(Napi::Env env, Object exports) { Function DbiClass = DefineClass(env, "Dbi", { // DbiWrap: Prepare constructor template // DbiWrap: Add functions to the prototype DbiWrap::InstanceMethod("close", &DbiWrap::close), DbiWrap::InstanceMethod("drop", &DbiWrap::drop), DbiWrap::InstanceMethod("stat", &DbiWrap::stat), }); exports.Set("Dbi", DbiClass); EXPORT_NAPI_FUNCTION("directWrite", directWrite); EXPORT_NAPI_FUNCTION("getByBinary", getByBinary); EXPORT_NAPI_FUNCTION("prefetch", prefetchNapi); EXPORT_NAPI_FUNCTION("getStringByBinary", getStringByBinary); EXPORT_NAPI_FUNCTION("getSharedByBinary", getSharedByBinary); EXPORT_FUNCTION_ADDRESS("getByBinaryPtr", getByBinaryFFI); // TODO: wrap mdb_stat too } // This file contains code from the node-lmdb project // Copyright (c) 2013-2017 Timur Kristóf // Copyright (c) 2021 Kristopher Tate // Licensed to you under the terms of the MIT license // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal // in the Software without restriction, including without limitation the rights // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell // copies of the Software, and to permit persons to whom the Software is // furnished to do so, subject to the following conditions: // The above copyright notice and this permission notice shall be included in // all copies or substantial portions of the Software. // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN // THE SOFTWARE.