commit fe2d14c1869c126f33dfc22584ea9caa61616fad Author: kangkang520 Date: Thu Nov 29 14:58:19 2018 +0800 初提交 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c3829c0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +/node_modules +/package-lock.json +/dist +/typings \ No newline at end of file diff --git a/.npmignore b/.npmignore new file mode 100644 index 0000000..11a9e76 --- /dev/null +++ b/.npmignore @@ -0,0 +1,5 @@ +/node_modules +/src +/example +/package-lock.json +/tsconfig.json \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..8add4e2 --- /dev/null +++ b/README.md @@ -0,0 +1,47 @@ +# cluster-memdb + +A key-value memory database for single process application and cluster application. + +Using this util, you can share datas in every processes of cluster. + +# installation +``` +npm install cluster-memdb +``` + +# usage +this example show you how to use it, you can read `.d.ts` for more information +```typescript +import {memdb, dbs} from 'cluster-memdb' + +// declare database (won't create any keys) +const userdb = memdb('user', 'id') +const lovedb = memdb('love', 'key') + +// create "user" key and write those 2 users to "user" +await userdb.save([ + {id:1, name:'Mike', gender:'male', love:['bread']}, + {id:2, name:'Louis', gender:'female', love:['beef', 'bread']}, +]) + +// clean "love" key and write new datas +await lovedb.replace([ + {key:'bread', price:'3.00'}, + {key:'beef', price:'10.00'} +]) + +await dbs() // ["user", "love"] + +await userdb.getAll() // [{id:1, ...}, {id:2, ...}] + +await lovedb.getByKeys(['bread']) // [{key:'bread', price:'3.00'}] + +await userdb.find({name:'Mike'}) // [{id:1, ...}] + +// "Mike" and "Louis" both deleted +await userdb.delete([1, 2]) + +// "bread" and "beef" both deleted +await love.clean() + +``` \ No newline at end of file diff --git a/example/e1.ts b/example/e1.ts new file mode 100644 index 0000000..b2049e8 --- /dev/null +++ b/example/e1.ts @@ -0,0 +1,25 @@ +import { memdb } from '../src' +import cluster from 'cluster' + +if (cluster.isMaster) { + for (let i = 0; i < 1; i++) { + cluster.fork() + } + setInterval(() => { + memdb('users', 'id').getAll().then(res => console.log(res)) + }, 1000) +} + +else { + const sleep = () => new Promise(resolve => setTimeout(resolve, 2000)); + const db = memdb('users', 'id'); + (async () => { + await db.save([{ id: 1, name: '小明1' }]) + await sleep() + await db.save([{ id: 2, name: '小明2' }]) + await sleep() + await db.save([{ id: 3, name: '小明3' }]) + await sleep() + await db.replace([{ id: 4, name: '小明4' }]) + })() +} \ No newline at end of file diff --git a/example/e2.ts b/example/e2.ts new file mode 100644 index 0000000..e8d65df --- /dev/null +++ b/example/e2.ts @@ -0,0 +1,17 @@ +import { memdb } from '../src' + +setInterval(() => { + memdb('users', 'id').getAll().then(res => console.log(res)) +}, 1000) + +const sleep = () => new Promise(resolve => setTimeout(resolve, 2000)); +const db = memdb('users', 'id'); +(async () => { + await db.save([{ id: 1, name: '小明1' }]) + await sleep() + await db.save([{ id: 2, name: '小明2' }]) + await sleep() + await db.save([{ id: 3, name: '小明3' }]) + await sleep() + await db.replace([{ id: 4, name: '小明4' }]) +})() diff --git a/package.json b/package.json new file mode 100644 index 0000000..be8abc6 --- /dev/null +++ b/package.json @@ -0,0 +1,30 @@ +{ + "name": "cluster-memdb", + "version": "1.0.3", + "description": "key-value memory database for single process application and cluster application", + "main": "dist/index.js", + "types": "typings/index.d.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "repository": { + "type": "git", + "url": "https://github.com/kangkang520/cluster-memdb.git" + }, + "keywords": [ + "memdb", + "cluster", + "memcache", + "memory", + "database", + "db", + "cache", + "share", + "message" + ], + "author": "", + "license": "ISC", + "devDependencies": { + "@types/node": "^10.12.10" + } +} diff --git a/src/db.ts b/src/db.ts new file mode 100644 index 0000000..2be1ac5 --- /dev/null +++ b/src/db.ts @@ -0,0 +1,209 @@ +import cluster from 'cluster' +import { IStorageDataItem, storage, IStorageDocument, dbs as _dbs } from './storage' + +const MESSAGE_TYPE_NAME = 'yz_memdb' +const sendable = cluster.isWorker && process.send + +type DataTypes = keyof ReturnType | 'dbs' + +let maxId = 1 + +//ID消息监听器(用于子进程) +const idListeners: { [k: number]: (data: any, error?: string) => void } = {} + +//主进程 +if (cluster.isMaster) { + //监听消息 + cluster.on('message', async (worker, message) => { + let res!: { id: number, data: any, db: string, keyName: string, type: DataTypes, messageType: string } + let err: Error = undefined! + let result: any = null //处理结果 + //解码数据得到结果 + try { + res = JSON.parse(message) + } catch (_err) { + err = _err + } + //这些不用处理 + if (err || res.messageType !== MESSAGE_TYPE_NAME) return + //针对不同消息进行处理 + try { + const db = memdb(res.db, res.keyName) + //获取数据库列表 + if (res.type == 'dbs') result = dbs() + //保存数据 + else if (res.type == 'save') await db.save(res.data) + //清空数据 + else if (res.type == 'clean') await db.clean() + //删除数据 + else if (res.type == 'delete') await db.delete(res.data) + //替换数据 + else if (res.type == 'replace') await db.replace(res.data) + //获取列表 + else if (res.type == 'getAll') result = await db.getAll() + //通过键获取数据 + else if (res.type == 'getByKeys') result = await db.getByKeys(res.data) + //获取键列表 + else if (res.type == 'keys') result = await db.keys() + //数据查询 + else if (res.type == 'find') result = await db.find(res.data) + //反馈消息给客户端 + worker.send(JSON.stringify({ id: res.id, messageType: MESSAGE_TYPE_NAME, data: result, type: res.type })) + } + //错误反馈 + catch (err) { + worker.send(JSON.stringify({ id: res.id, messageType: MESSAGE_TYPE_NAME, error: err.message, type: res.type })) + } + }) +} +//子进程 +else { + process.on('message', message => { + let id!: number + let data!: any + let error!: string + let messageType!: string + let err: Error = undefined! + //解码数据得到结果 + try { + const res = JSON.parse(message) + id = res.id + data = res.data + error = res.error + messageType = res.messageType + } catch (_err) { + err = _err + } + //过滤无用数据 + if (err || messageType !== MESSAGE_TYPE_NAME) return + //取得消息监听器 + const listener = idListeners[id] + if (!listener) return + //调用监听器并删除之 + listener(data, error) + delete idListeners[id] + }) +} + +//发送数据并得到结果 +function send(type: DataTypes, db: string, keyName: string, data: any): Promise { + const id = maxId++ + return new Promise((resolve, reject) => { + //监听消息 + idListeners[id] = (data, error) => { + if (error) reject(new Error(error)) + else resolve(data) + } + //发送数据 + process.send!(JSON.stringify({ id, type, messageType: MESSAGE_TYPE_NAME, data, db, keyName })) + }) +} + + +/** + * 创建内存数据库 + * @param db 数据库名 + * @param keyName 键名称,如id + */ +export function memdb(db: string, keyName: string) { + //是否允许发送数据 + const s = storage(db) + + /** + * 保存值 + * @param vals 要保存的值 + */ + async function save(vals: Array | IStorageDataItem): Promise { + if (sendable) return await send('save', db, keyName, vals) + else ((vals instanceof Array) ? vals : [vals]).forEach(val => { + const key = val[keyName] + key && s.save(key, val) + }) + } + + /** + * 清空数据 + */ + async function clean() { + if (sendable) return await send('clean', db, keyName, null) + else s.clean() + } + + /** + * 清空原有值,并保存新值 + * @param vals 要保存的值 + */ + async function replace(vals: Array) { + if (sendable) return await send('replace', db, keyName, vals) + else { + s.clean() + vals.forEach(val => { + const key = val[keyName] + key && s.save(key, val) + }) + } + } + + /** + * 删除数据 + * @param keys 要删除的键列表 + */ + async function _delete(keys: Array): Promise { + if (sendable) return await send('delete', db, keyName, keys) + else s.delete(keys) + } + + /** + * 获取所有数据 + */ + async function getAll(): Promise | null> { + if (sendable) return await send('getAll', db, keyName, null) + else return s.all() + } + + /** + * 通过指定的键获取数据 + * @param keys 键列表 + */ + async function getByKeys(keys: Array): Promise | null> { + if (sendable) return await send('getByKeys', db, keyName, keys) + else return s.keysof(keys) + } + + /** + * 数据查询 + * @param option 查询选项 + */ + async function find(option: { [i: string]: string | number | boolean }): Promise | null> { + if (sendable) return await send('find', db, keyName, option) + else return s.find(option) + } + + /** + * 获取所有键列表 + */ + async function keys(): Promise> { + if (sendable) return await send('keys', db, keyName, null) + else return Object.keys(s.datas() || {}) + } + + //返回结果 + return { + save, + clean, + replace, + delete: _delete, + getAll, + getByKeys, + find, + keys, + } +} + +/** + * 获取所有数据库列表 + */ +export async function dbs(): Promise> { + if (sendable) return await send('delete', null!, null!, null) + else return _dbs() +} \ No newline at end of file diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..33d039e --- /dev/null +++ b/src/index.ts @@ -0,0 +1,4 @@ +import { memdb } from './db' + +export { memdb } +module.exports = { memdb } \ No newline at end of file diff --git a/src/storage.ts b/src/storage.ts new file mode 100644 index 0000000..258912d --- /dev/null +++ b/src/storage.ts @@ -0,0 +1,176 @@ +/** + * @author 陆益之 + * 此文件用于处理数据存储,数据存储共分为3个部分,库、文档、数据,每个库下面有多个文档,每个文档下面有多个数据,例如: + * [users] + * [lee] + * [name = lee] + * [age = 20] + * [gender = male] + * [love = [1,2]] + * [jane] + * [name = jane] + * [age=19] + * [gender = female] + * [love = [2]] + * [foods] + * [1] + * [id = 1] + * [name = beef] + * [2] + * [id = 2] + * [name = bread] + */ + +/** 数据 */ +export type IStorageDataItem = { [i: string]: any } + +/** 文档(键=>值) */ +export type IStorageDocument = { [i: string]: IStorageDataItem } + +/** 库 */ +export type IStorageDB = { [i: string]: IStorageDocument } + + +/** 数据存储 */ +const storages: IStorageDB = {} + +/** + * 建立一个库操作器 + * @param db 库名称 + */ +export function storage(db: string) { + /** + * 是否存在数据库 + */ + function exists(): boolean { + return !!storages[db] + } + + /** + * 删除数据库 + */ + function drop() { + delete storages[db] + } + + /** + * 重命名数据库并返回新名称的数据库 + * @param newName 新数据库名称 + */ + function rename(newName: string) { + storages[newName] = storages[db] + drop() + return storage(newName) + } + + /** + * 清空数据 + */ + function clean() { + storages[db] = {} + } + + /** + * 获取数据列表 + * @param init 如果没有是否初始化 + */ + function datas(init?: boolean): IStorageDocument | null { + if (init && !storages[db]) storages[db] = {} + return storages[db] || null + } + + /** + * 通过键获取数据 + * @param keys 键列表 + */ + function keysof(keys: Array) { + const d = datas() + if (!d) return null + const buffer: Array = [] + keys.forEach(key => { + if (d[key] === undefined) return + buffer.push(d[key]) + }) + return buffer + } + + /** + * 获取所有数据 + */ + function all() { + const d = datas() + return d ? Object.values(d) : null + } + + /** + * 数据查询 + * @param option 查询选项 + */ + function find(option: { [key: string]: string | boolean | number }) { + const d = datas() + if (!d) return null + const okeys = Object.keys(option) + //如果没有任何选项则返回所有数据 + if (!okeys.length) return all() + //否则按照选项查询 + const buffer: Array = [] + for (let key in d) { + let val = d[key] + if (!val) continue + let eq = true + for (let okey in okeys) { + if (option[okey] !== val[okey]) { + eq = false + break + } + } + if (!eq) continue + buffer.push(val) + } + return buffer + } + + /** + * 删除数据 + * @param keys 要删除的数据的键 + */ + function _delete(keys: Array) { + const d = datas() + if (!d) return + keys.forEach(key => { + delete d[key] + }) + } + + /** + * 保存数据,无则添加 + * @param key 要修改的数据的键 + * @param value 数据值 + * @param replace 是否进行替换操作,默认true + */ + function save(key: string, value: IStorageDataItem, replace: boolean = true) { + const d = datas(true)! + //没有数据或者说是替换则直接赋值 + if (d[key] || replace) { + d[key] = value + return + } + //按键覆盖 + const data = d[key] + Object.keys(value).forEach(key => data[key] = value[key]) + } + + //返回操作函数 + return { + exists, drop, rename, clean, + datas, keysof, all, find, + delete: _delete, save, + } +} + +/** + * 获取所有库的键 + */ +export function dbs() { + return Object.keys(storages) +} \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..e17d7ef --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,64 @@ +{ + "compilerOptions": { + /* Basic Options */ + "target": "esnext", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017','ES2018' or 'ESNEXT'. */ + "module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */ + "lib": [ + "es2015", + "esnext" + ], /* Specify library files to be included in the compilation. */ + // "allowJs": true, /* Allow javascript files to be compiled. */ + // "checkJs": true, /* Report errors in .js files. */ + // "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */ + "declaration": true, /* Generates corresponding '.d.ts' file. */ + "declarationDir": "typings", /* Generates corresponding '.d.ts' file. */ + // "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */ + // "sourceMap": true, /* Generates corresponding '.map' file. */ + // "outFile": "./", /* Concatenate and emit output to single file. */ + "outDir": "./dist", /* Redirect output structure to the directory. */ + "rootDir": "./src", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */ + // "composite": true, /* Enable project compilation */ + // "removeComments": true, /* Do not emit comments to output. */ + // "noEmit": true, /* Do not emit outputs. */ + // "importHelpers": true, /* Import emit helpers from 'tslib'. */ + // "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */ + // "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */ + /* Strict Type-Checking Options */ + "strict": true, /* Enable all strict type-checking options. */ + // "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */ + // "strictNullChecks": true, /* Enable strict null checks. */ + // "strictFunctionTypes": true, /* Enable strict checking of function types. */ + // "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */ + // "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */ + // "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */ + /* Additional Checks */ + // "noUnusedLocals": true, /* Report errors on unused locals. */ + // "noUnusedParameters": true, /* Report errors on unused parameters. */ + // "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */ + // "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */ + /* Module Resolution Options */ + // "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */ + // "baseUrl": "./", /* Base directory to resolve non-absolute module names. */ + // "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */ + // "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */ + // "typeRoots": [], /* List of folders to include type definitions from. */ + // "types": [], /* Type declaration files to be included in compilation. */ + // "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */ + "esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */ + // "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */ + /* Source Map Options */ + // "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */ + // "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */ + // "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */ + // "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */ + /* Experimental Options */ + // "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */ + // "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */ + }, + "include": [ + "src/**/*.ts" + ], + "exclude": [ + "node_modules" + ] +} \ No newline at end of file