yizhi-js-lib/cluster-memdb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

初提交

Browse Source
This commit is contained in:
kangkang520
2018-11-29 14:58:19 +08:00
commit fe2d14c186
10 changed files with 581 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored Normal file
View File

@ -0,0 +1,4 @@
/node_modules
/package-lock.json
/dist
/typings

5
.npmignore Normal file
View File

@ -0,0 +1,5 @@
/node_modules
/src
/example
/package-lock.json
/tsconfig.json

47
README.md Normal file
View File

@ -0,0 +1,47 @@
# cluster-memdb
A key-value memory database for single process application and cluster application.
Using this util, you can share datas in every processes of cluster.
# installation
```
npm install cluster-memdb
```
# usage
this example show you how to use it, you can read `.d.ts` for more information
```typescript
import {memdb, dbs} from 'cluster-memdb'
// declare database (won't create any keys)
const userdb = memdb('user', 'id')
const lovedb = memdb('love', 'key')
// create "user" key and write those 2 users to "user"
await userdb.save([
{id:1, name:'Mike', gender:'male', love:['bread']},
{id:2, name:'Louis', gender:'female', love:['beef', 'bread']},
])
// clean "love" key and write new datas
await lovedb.replace([
{key:'bread', price:'3.00'},
{key:'beef', price:'10.00'}
])
await dbs() // ["user", "love"]
await userdb.getAll() // [{id:1, ...}, {id:2, ...}]
await lovedb.getByKeys(['bread']) // [{key:'bread', price:'3.00'}]
await userdb.find({name:'Mike'}) // [{id:1, ...}]
// "Mike" and "Louis" both deleted
await userdb.delete([1, 2])
// "bread" and "beef" both deleted
await love.clean()
```

25
example/e1.ts Normal file
View File

@ -0,0 +1,25 @@
import { memdb } from '../src'
import cluster from 'cluster'
if (cluster.isMaster) {
for (let i = 0; i < 1; i++) {
cluster.fork()
}
setInterval(() => {
memdb('users', 'id').getAll().then(res => console.log(res))
}, 1000)
}
else {
const sleep = () => new Promise(resolve => setTimeout(resolve, 2000));
const db = memdb('users', 'id');
(async () => {
await db.save([{ id: 1, name: '小明1' }])
await sleep()
await db.save([{ id: 2, name: '小明2' }])
await sleep()
await db.save([{ id: 3, name: '小明3' }])
await sleep()
await db.replace([{ id: 4, name: '小明4' }])
})()
}

17
example/e2.ts Normal file
View File

@ -0,0 +1,17 @@
import { memdb } from '../src'
setInterval(() => {
memdb('users', 'id').getAll().then(res => console.log(res))
}, 1000)
const sleep = () => new Promise(resolve => setTimeout(resolve, 2000));
const db = memdb('users', 'id');
(async () => {
await db.save([{ id: 1, name: '小明1' }])
await sleep()
await db.save([{ id: 2, name: '小明2' }])
await sleep()
await db.save([{ id: 3, name: '小明3' }])
await sleep()
await db.replace([{ id: 4, name: '小明4' }])
})()

30
package.json Normal file
View File

@ -0,0 +1,30 @@
{
"name": "cluster-memdb",
"version": "1.0.3",
"description": "key-value memory database for single process application and cluster application",
"main": "dist/index.js",
"types": "typings/index.d.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"repository": {
"type": "git",
"url": "https://github.com/kangkang520/cluster-memdb.git"
},
"keywords": [
"memdb",
"cluster",
"memcache",
"memory",
"database",
"db",
"cache",
"share",
"message"
],
"author": "",
"license": "ISC",
"devDependencies": {
"@types/node": "^10.12.10"
}
}

209
src/db.ts Normal file
View File

@ -0,0 +1,209 @@
import cluster from 'cluster'
import { IStorageDataItem, storage, IStorageDocument, dbs as _dbs } from './storage'
const MESSAGE_TYPE_NAME = 'yz_memdb'
const sendable = cluster.isWorker && process.send
type DataTypes = keyof ReturnType<typeof memdb> | 'dbs'
let maxId = 1
//ID消息监听器（用于子进程）
const idListeners: { [k: number]: (data: any, error?: string) => void } = {}
//主进程
if (cluster.isMaster) {
//监听消息
cluster.on('message', async (worker, message) => {
let res!: { id: number, data: any, db: string, keyName: string, type: DataTypes, messageType: string }
let err: Error = undefined!
let result: any = null //处理结果
//解码数据得到结果
try {
res = JSON.parse(message)
} catch (_err) {
err = _err
}
//这些不用处理
if (err || res.messageType !== MESSAGE_TYPE_NAME) return
//针对不同消息进行处理
try {
const db = memdb(res.db, res.keyName)
//获取数据库列表
if (res.type == 'dbs') result = dbs()
//保存数据
else if (res.type == 'save') await db.save(res.data)
//清空数据
else if (res.type == 'clean') await db.clean()
//删除数据
else if (res.type == 'delete') await db.delete(res.data)
//替换数据
else if (res.type == 'replace') await db.replace(res.data)
//获取列表
else if (res.type == 'getAll') result = await db.getAll()
//通过键获取数据
else if (res.type == 'getByKeys') result = await db.getByKeys(res.data)
//获取键列表
else if (res.type == 'keys') result = await db.keys()
//数据查询
else if (res.type == 'find') result = await db.find(res.data)
//反馈消息给客户端
worker.send(JSON.stringify({ id: res.id, messageType: MESSAGE_TYPE_NAME, data: result, type: res.type }))
}
//错误反馈
catch (err) {
worker.send(JSON.stringify({ id: res.id, messageType: MESSAGE_TYPE_NAME, error: err.message, type: res.type }))
}
})
}
//子进程
else {
process.on('message', message => {
let id!: number
let data!: any
let error!: string
let messageType!: string
let err: Error = undefined!
//解码数据得到结果
try {
const res = JSON.parse(message)
id = res.id
data = res.data
error = res.error
messageType = res.messageType
} catch (_err) {
err = _err
}
//过滤无用数据
if (err || messageType !== MESSAGE_TYPE_NAME) return
//取得消息监听器
const listener = idListeners[id]
if (!listener) return
//调用监听器并删除之
listener(data, error)
delete idListeners[id]
})
}
//发送数据并得到结果
function send(type: DataTypes, db: string, keyName: string, data: any): Promise<any> {
const id = maxId++
return new Promise((resolve, reject) => {
//监听消息
idListeners[id] = (data, error) => {
if (error) reject(new Error(error))
else resolve(data)
}
//发送数据
process.send!(JSON.stringify({ id, type, messageType: MESSAGE_TYPE_NAME, data, db, keyName }))
})
}
/**
* 创建内存数据库
* @param db 数据库名
* @param keyName 键名称，如id
*/
export function memdb(db: string, keyName: string) {
//是否允许发送数据
const s = storage(db)
/**
* 保存值
* @param vals 要保存的值
*/
async function save(vals: Array<IStorageDataItem> | IStorageDataItem): Promise<void> {
if (sendable) return await send('save', db, keyName, vals)
else ((vals instanceof Array) ? vals : [vals]).forEach(val => {
const key = val[keyName]
key && s.save(key, val)
})
}
/**
* 清空数据
*/
async function clean() {
if (sendable) return await send('clean', db, keyName, null)
else s.clean()
}
/**
* 清空原有值，并保存新值
* @param vals 要保存的值
*/
async function replace(vals: Array<IStorageDataItem>) {
if (sendable) return await send('replace', db, keyName, vals)
else {
s.clean()
vals.forEach(val => {
const key = val[keyName]
key && s.save(key, val)
})
}
}
/**
* 删除数据
* @param keys 要删除的键列表
*/
async function _delete(keys: Array<string>): Promise<void> {
if (sendable) return await send('delete', db, keyName, keys)
else s.delete(keys)
}
/**
* 获取所有数据
*/
async function getAll(): Promise<Array<IStorageDataItem> | null> {
if (sendable) return await send('getAll', db, keyName, null)
else return s.all()
}
/**
* 通过指定的键获取数据
* @param keys 键列表
*/
async function getByKeys(keys: Array<string>): Promise<Array<IStorageDataItem> | null> {
if (sendable) return await send('getByKeys', db, keyName, keys)
else return s.keysof(keys)
}
/**
* 数据查询
* @param option 查询选项
*/
async function find(option: { [i: string]: string | number | boolean }): Promise<Array<IStorageDataItem> | null> {
if (sendable) return await send('find', db, keyName, option)
else return s.find(option)
}
/**
* 获取所有键列表
*/
async function keys(): Promise<Array<string>> {
if (sendable) return await send('keys', db, keyName, null)
else return Object.keys(s.datas() || {})
}
//返回结果
return {
save,
clean,
replace,
delete: _delete,
getAll,
getByKeys,
find,
keys,
}
}
/**
* 获取所有数据库列表
*/
export async function dbs(): Promise<Array<string>> {
if (sendable) return await send('delete', null!, null!, null)
else return _dbs()
}

4
src/index.ts Normal file
View File

@ -0,0 +1,4 @@
import { memdb } from './db'
export { memdb }
module.exports = { memdb }

176
src/storage.ts Normal file
View File

@ -0,0 +1,176 @@
/**
* @author 陆益之
* 此文件用于处理数据存储，数据存储共分为3个部分，库、文档、数据，每个库下面有多个文档，每个文档下面有多个数据，例如：
* [users]
* [lee]
* [name = lee]
* [age = 20]
* [gender = male]
* [love = [1,2]]
* [jane]
* [name = jane]
* [age=19]
* [gender = female]
* [love = [2]]
* [foods]
* [1]
* [id = 1]
* [name = beef]
* [2]
* [id = 2]
* [name = bread]
*/
/** 数据 */
export type IStorageDataItem = { [i: string]: any }
/** 文档（键=>值） */
export type IStorageDocument = { [i: string]: IStorageDataItem }
/** 库 */
export type IStorageDB = { [i: string]: IStorageDocument }
/** 数据存储 */
const storages: IStorageDB = {}
/**
* 建立一个库操作器
* @param db 库名称
*/
export function storage(db: string) {
/**
* 是否存在数据库
*/
function exists(): boolean {
return !!storages[db]
}
/**
* 删除数据库
*/
function drop() {
delete storages[db]
}
/**
* 重命名数据库并返回新名称的数据库
* @param newName 新数据库名称
*/
function rename(newName: string) {
storages[newName] = storages[db]
drop()
return storage(newName)
}
/**
* 清空数据
*/
function clean() {
storages[db] = {}
}
/**
* 获取数据列表
* @param init 如果没有是否初始化
*/
function datas(init?: boolean): IStorageDocument | null {
if (init && !storages[db]) storages[db] = {}
return storages[db] || null
}
/**
* 通过键获取数据
* @param keys 键列表
*/
function keysof(keys: Array<string>) {
const d = datas()
if (!d) return null
const buffer: Array<IStorageDataItem> = []
keys.forEach(key => {
if (d[key] === undefined) return
buffer.push(d[key])
})
return buffer
}
/**
* 获取所有数据
*/
function all() {
const d = datas()
return d ? Object.values(d) : null
}
/**
* 数据查询
* @param option 查询选项
*/
function find(option: { [key: string]: string | boolean | number }) {
const d = datas()
if (!d) return null
const okeys = Object.keys(option)
//如果没有任何选项则返回所有数据
if (!okeys.length) return all()
//否则按照选项查询
const buffer: Array<IStorageDataItem> = []
for (let key in d) {
let val = d[key]
if (!val) continue
let eq = true
for (let okey in okeys) {
if (option[okey] !== val[okey]) {
eq = false
break
}
}
if (!eq) continue
buffer.push(val)
}
return buffer
}
/**
* 删除数据
* @param keys 要删除的数据的键
*/
function _delete(keys: Array<string>) {
const d = datas()
if (!d) return
keys.forEach(key => {
delete d[key]
})
}
/**
* 保存数据，无则添加
* @param key 要修改的数据的键
* @param value 数据值
* @param replace 是否进行替换操作，默认true
*/
function save(key: string, value: IStorageDataItem, replace: boolean = true) {
const d = datas(true)!
//没有数据或者说是替换则直接赋值
if (d[key] || replace) {
d[key] = value
return
}
//按键覆盖
const data = d[key]
Object.keys(value).forEach(key => data[key] = value[key])
}
//返回操作函数
return {
exists, drop, rename, clean,
datas, keysof, all, find,
delete: _delete, save,
}
}
/**
* 获取所有库的键
*/
export function dbs() {
return Object.keys(storages)
}

64
tsconfig.json Normal file
View File

@ -0,0 +1,64 @@
{
"compilerOptions": {
/* Basic Options */
"target": "esnext", /* Specify ECMAScript target version: 'ES3' (default), 'ES5', 'ES2015', 'ES2016', 'ES2017','ES2018' or 'ESNEXT'. */
"module": "commonjs", /* Specify module code generation: 'none', 'commonjs', 'amd', 'system', 'umd', 'es2015', or 'ESNext'. */
"lib": [
"es2015",
"esnext"
], /* Specify library files to be included in the compilation. */
// "allowJs": true, /* Allow javascript files to be compiled. */
// "checkJs": true, /* Report errors in .js files. */
// "jsx": "preserve", /* Specify JSX code generation: 'preserve', 'react-native', or 'react'. */
"declaration": true, /* Generates corresponding '.d.ts' file. */
"declarationDir": "typings", /* Generates corresponding '.d.ts' file. */
// "declarationMap": true, /* Generates a sourcemap for each corresponding '.d.ts' file. */
// "sourceMap": true, /* Generates corresponding '.map' file. */
// "outFile": "./", /* Concatenate and emit output to single file. */
"outDir": "./dist", /* Redirect output structure to the directory. */
"rootDir": "./src", /* Specify the root directory of input files. Use to control the output directory structure with --outDir. */
// "composite": true, /* Enable project compilation */
// "removeComments": true, /* Do not emit comments to output. */
// "noEmit": true, /* Do not emit outputs. */
// "importHelpers": true, /* Import emit helpers from 'tslib'. */
// "downlevelIteration": true, /* Provide full support for iterables in 'for-of', spread, and destructuring when targeting 'ES5' or 'ES3'. */
// "isolatedModules": true, /* Transpile each file as a separate module (similar to 'ts.transpileModule'). */
/* Strict Type-Checking Options */
"strict": true, /* Enable all strict type-checking options. */
// "noImplicitAny": true, /* Raise error on expressions and declarations with an implied 'any' type. */
// "strictNullChecks": true, /* Enable strict null checks. */
// "strictFunctionTypes": true, /* Enable strict checking of function types. */
// "strictPropertyInitialization": true, /* Enable strict checking of property initialization in classes. */
// "noImplicitThis": true, /* Raise error on 'this' expressions with an implied 'any' type. */
// "alwaysStrict": true, /* Parse in strict mode and emit "use strict" for each source file. */
/* Additional Checks */
// "noUnusedLocals": true, /* Report errors on unused locals. */
// "noUnusedParameters": true, /* Report errors on unused parameters. */
// "noImplicitReturns": true, /* Report error when not all code paths in function return a value. */
// "noFallthroughCasesInSwitch": true, /* Report errors for fallthrough cases in switch statement. */
/* Module Resolution Options */
// "moduleResolution": "node", /* Specify module resolution strategy: 'node' (Node.js) or 'classic' (TypeScript pre-1.6). */
// "baseUrl": "./", /* Base directory to resolve non-absolute module names. */
// "paths": {}, /* A series of entries which re-map imports to lookup locations relative to the 'baseUrl'. */
// "rootDirs": [], /* List of root folders whose combined content represents the structure of the project at runtime. */
// "typeRoots": [], /* List of folders to include type definitions from. */
// "types": [], /* Type declaration files to be included in compilation. */
// "allowSyntheticDefaultImports": true, /* Allow default imports from modules with no default export. This does not affect code emit, just typechecking. */
"esModuleInterop": true /* Enables emit interoperability between CommonJS and ES Modules via creation of namespace objects for all imports. Implies 'allowSyntheticDefaultImports'. */
// "preserveSymlinks": true, /* Do not resolve the real path of symlinks. */
/* Source Map Options */
// "sourceRoot": "", /* Specify the location where debugger should locate TypeScript files instead of source locations. */
// "mapRoot": "", /* Specify the location where debugger should locate map files instead of generated locations. */
// "inlineSourceMap": true, /* Emit a single file with source maps instead of having a separate file. */
// "inlineSources": true, /* Emit the source alongside the sourcemaps within a single file; requires '--inlineSourceMap' or '--sourceMap' to be set. */
/* Experimental Options */
// "experimentalDecorators": true, /* Enables experimental support for ES7 decorators. */
// "emitDecoratorMetadata": true, /* Enables experimental support for emitting type metadata for decorators. */
},
"include": [
"src/**/*.ts"
],
"exclude": [
"node_modules"
]
}