From cc10c8b7592708e8b96e2d4c4be87b07c48b5432 Mon Sep 17 00:00:00 2001 From: Kyriakos Barbounakis Date: Wed, 15 Jan 2025 12:31:53 +0200 Subject: [PATCH 1/4] add db caching strategy --- spec/MemoryCacheStrategy.ts | 81 +++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 spec/MemoryCacheStrategy.ts diff --git a/spec/MemoryCacheStrategy.ts b/spec/MemoryCacheStrategy.ts new file mode 100644 index 0000000..a08c634 --- /dev/null +++ b/spec/MemoryCacheStrategy.ts @@ -0,0 +1,81 @@ +import { DataApplication } from "../data-application"; +import { DataCacheStrategy } from "../data-cache"; +import { SqliteAdapter } from "@themost/sqlite"; +import { DataConfigurationStrategy, SchemaLoaderStrategy } from "../data-configuration"; +import path from 'path'; +import { ConfigurationBase, DataModelProperties } from "@themost/common"; + +class MemoryCacheApplication extends DataApplication { + constructor() { + super(path.resolve(process.cwd(), '.cache')); + const config = this.configuration.getStrategy(DataConfigurationStrategy); + config.adapterTypes.set('sqlite', { + name: 'sqlite', + invariantName: 'sqlite', + type: SqliteAdapter + }); + config.adapters.push({ + name: 'cache', + invariantName: 'sqlite', + options: { + database: ':memory:' + } + }); + const schema = this.getService(SchemaLoaderStrategy); + schema.setModelDefinition({ + name: 'MemoryCache', + title: 'Memory Cache', + caching: 'none', + version: '0.0.0', + fields: [ + { + name: 'id', + type: 'Text', + primary: true, + nullable: false + }, + { + name: 'additionalType', + type: 'Text' + }, + { + name: 'value', + type: 'Text' + }, + { + name: 'expiresAt', + type: 'DateTime' + } + ] + } as DataModelProperties) + } +} + +class MemoryCacheStrategy extends DataCacheStrategy { + + protected cache: MemoryCacheApplication; + + constructor(config: ConfigurationBase) { + super(config); + this.cache = new MemoryCacheApplication(); + } + + get(key: string): Promise { + throw new Error("Method not implemented."); + } + + add(key: string, value: any, absoluteExpiration?: number): Promise { + throw new Error("Method not implemented."); + } + remove(key: string): Promise { + throw new Error("Method not implemented."); + } + clear(): Promise { + throw new Error("Method not implemented."); + } + getOrDefault(key: string, getFunc: () => Promise, absoluteExpiration?: number): Promise { + throw new Error("Method not implemented."); + } +} + +export { MemoryCacheStrategy }; \ No newline at end of file From 100c3f933ef750b4da11be0008cc69182c666922 Mon Sep 17 00:00:00 2001 From: Kyriakos Barbounakis Date: Wed, 15 Jan 2025 12:32:11 +0200 Subject: [PATCH 2/4] add database caching strategy --- .eslintrc | 2 +- OnJsonAttribute.js | 42 +++++- data-context.js | 4 +- data-listeners.js | 6 +- data-model.js | 18 ++- jest.config.js | 3 + package-lock.json | 43 +++++- package.json | 1 + platform-server/CacheEntry.js | 151 +++++++++++++++++++ platform-server/MemoryCacheStrategy.d.ts | 9 ++ platform-server/MemoryCacheStrategy.js | 181 +++++++++++++++++++++++ platform-server/index.d.ts | 1 + platform-server/index.js | 4 + platform-server/package.json | 10 ++ spec/MemoryCacheEntry.spec.ts | 37 +++++ spec/MemoryCacheStrategy.ts | 81 ---------- tsconfig.json | 3 + 17 files changed, 500 insertions(+), 96 deletions(-) create mode 100644 platform-server/CacheEntry.js create mode 100644 platform-server/MemoryCacheStrategy.d.ts create mode 100644 platform-server/MemoryCacheStrategy.js create mode 100644 platform-server/index.d.ts create mode 100644 platform-server/index.js create mode 100644 platform-server/package.json create mode 100644 spec/MemoryCacheEntry.spec.ts delete mode 100644 spec/MemoryCacheStrategy.ts diff --git a/.eslintrc b/.eslintrc index c0e0751..3a85d71 100644 --- a/.eslintrc +++ b/.eslintrc @@ -4,7 +4,7 @@ "plugin:node/recommended" ], "parserOptions": { - "ecmaVersion": 2015, + "ecmaVersion": 2018, "sourceType": "module" }, "env": { diff --git a/OnJsonAttribute.js b/OnJsonAttribute.js index 4c24635..1de331b 100644 --- a/OnJsonAttribute.js +++ b/OnJsonAttribute.js @@ -168,14 +168,52 @@ class OnJsonAttribute { * @returns void */ static afterSelect(event, callback) { - const jsonAttributes = event.model.attributes.filter((attr) => { - return attr.type === 'Json' && attr.additionalType != null && attr.model === event.model.name; + const anyJsonAttributes = event.model.attributes.filter((attr) => { + return attr.type === 'Json' && attr.model === event.model.name; + }); + if (anyJsonAttributes.length === 0) { + return callback(); + } + const jsonAttributes = anyJsonAttributes.filter((attr) => { + return attr.additionalType != null; }).map((attr) => { return attr.name }); + // if there are no json attributes with additional type if (jsonAttributes.length === 0) { + // get json attributes with no additional type + const unknownJsonAttributes = anyJsonAttributes.filter((attr) => { + return attr.additionalType == null; + }).map((attr) => { + return attr.name + }); + // parse json for each item + if (unknownJsonAttributes.length > 0) { + const parseUnknownJson = (item) => { + unknownJsonAttributes.forEach((name) => { + if (Object.prototype.hasOwnProperty.call(item, name)) { + const value = item[name]; + if (typeof value === 'string') { + item[name] = JSON.parse(value); + } + } + }); + }; + // iterate over result + const {result} = event; + if (result == null) { + return callback(); + } + if (Array.isArray(result)) { + result.forEach((item) => parseUnknownJson(item)); + } else { + // or parse json for single item + parseUnknownJson(result) + } + } return callback(); } + let select = []; const { viewAdapter: entity } = event.model; if (event.emitter && event.emitter.query && event.emitter.query.$select) { diff --git a/data-context.js b/data-context.js index 2faa93f..862a54d 100644 --- a/data-context.js +++ b/data-context.js @@ -7,6 +7,7 @@ var {DataConfigurationStrategy} = require('./data-configuration'); var cfg = require('./data-configuration'); var Symbol = require('symbol'); var nameProperty = Symbol('name'); +var { DataModel } = require('./data-model'); /** * @classdesc Represents the default data context of MOST Data Applications. @@ -171,8 +172,7 @@ DefaultDataContext.prototype.model = function(name) { var obj = self.getConfiguration().getStrategy(DataConfigurationStrategy).model(modelName); if (_.isNil(obj)) return null; - var DataModel = require('./data-model').DataModel, - model = new DataModel(obj); + var model = new DataModel(obj); //set model context model.context = self; //return model diff --git a/data-listeners.js b/data-listeners.js index 08f7b79..a491e09 100644 --- a/data-listeners.js +++ b/data-listeners.js @@ -12,6 +12,7 @@ var {TraceUtils} = require('@themost/common'); var {TextUtils} = require('@themost/common'); var {DataCacheStrategy} = require('./data-cache'); var {DataFieldQueryResolver} = require('./data-field-query-resolver'); +var functions = require('./functions'); /** * @classdesc Represents an event listener for validating not nullable fields. This listener is automatically registered in all data models. @@ -222,8 +223,7 @@ function CalculatedValueListener() { */ CalculatedValueListener.prototype.beforeSave = function(event, callback) { //get function context - var functions = require('./functions'), - functionContext = functions.createContext(); + var functionContext = functions.createContext(); _.assign(functionContext, event); functionContext.context = event.model.context; //find all attributes that have a default value @@ -555,7 +555,7 @@ DefaultValueListener.prototype.beforeSave = function(event, callback) { } else { //get function context - var functions = require('./functions'), functionContext = functions.createContext(); + var functionContext = functions.createContext(); _.assign(functionContext, event); //find all attributes that have a default value var attrs = event.model.attributes.filter(function(x) { return (typeof x.value!== 'undefined'); }); diff --git a/data-model.js b/data-model.js index 19e42c4..ea445a9 100644 --- a/data-model.js +++ b/data-model.js @@ -44,6 +44,7 @@ require('@themost/promise-sequence'); var DataObjectState = types.DataObjectState; var { OnJsonAttribute } = require('./OnJsonAttribute'); var { isObjectDeep } = require('./is-object'); +var { DataStateValidatorListener } = require('./data-state-validator'); /** * @this DataModel * @param {DataField} field @@ -616,8 +617,7 @@ function unregisterContextListeners() { var DataCachingListener = dataListeners.DataCachingListener; var DataModelCreateViewListener = dataListeners.DataModelCreateViewListener; var DataModelSeedListener = dataListeners.DataModelSeedListener; - var DataStateValidatorListener = require('./data-state-validator').DataStateValidatorListener; - + //1. State validator listener this.on('before.save', DataStateValidatorListener.prototype.beforeSave); this.on('before.remove', DataStateValidatorListener.prototype.beforeRemove); @@ -710,9 +710,10 @@ DataModel.prototype.join = function(model) { * @param {String|*} attr - A string that represents the name of a field * @returns DataQueryable */ +// eslint-disable-next-line no-unused-vars DataModel.prototype.where = function(attr) { var result = new DataQueryable(this); - return result.where(attr); + return result.where.apply(result, Array.from(arguments)); }; /** @@ -1508,7 +1509,13 @@ function cast_(obj, state) { if (mapping == null) { var {[name]: value} = obj; if (x.type === 'Json') { - result[x.name] = isObjectDeep(value) ? JSON.stringify(value) : null; + if (value != null) { + // set json value + result[x.name] = typeof value === 'string' ? value : JSON.stringify(value); + } else { + // set null value + result[x.name] = value; + } } else { result[x.name] = value; } @@ -1758,8 +1765,7 @@ DataModel.prototype.save = function(obj, callback) * @see DataObjectState */ DataModel.prototype.inferState = function(obj, callback) { - var self = this, - DataStateValidatorListener = require('./data-state-validator').DataStateValidatorListener; + var self = this; var e = { model:self, target:obj }; DataStateValidatorListener.prototype.beforeSave(e, function(err) { //if error return error diff --git a/jest.config.js b/jest.config.js index e284210..a929dd4 100644 --- a/jest.config.js +++ b/jest.config.js @@ -82,6 +82,9 @@ module.exports = { // A map from regular expressions to module names or to arrays of module names that allow to stub out resources with a single module moduleNameMapper: { + '^@themost/data/platform-server$': [ + '/platform-server/index' + ], '^@themost/data$': [ '/index' ] diff --git a/package-lock.json b/package-lock.json index fe8aa6a..8d7b705 100644 --- a/package-lock.json +++ b/package-lock.json @@ -27,6 +27,7 @@ "@themost/common": "^2.5.11", "@themost/json-logger": "^1.1.0", "@themost/peers": "^1.0.2", + "@themost/pool": "^2.10.1", "@themost/query": "^2.6.73", "@themost/sqlite": "^2.8.4", "@themost/xml": "^2.5.2", @@ -53,7 +54,7 @@ }, "peerDependencies": { "@themost/common": "^2.5.11", - "@themost/query": "^2.6.0", + "@themost/query": "lts", "@themost/xml": "^2.5.2" } }, @@ -5212,6 +5213,21 @@ "add-peers": "bin/add-peers" } }, + "node_modules/@themost/pool": { + "version": "2.10.1", + "resolved": "https://registry.npmjs.org/@themost/pool/-/pool-2.10.1.tgz", + "integrity": "sha512-KVn+Sh+b904a3DzWwVeilpBh3p4w/7ZpHXh1oY1ZyX4VLPPStrvA/rAX/Z49yYlRWWAiPbqtoY1jFhhBGu72Kg==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "generic-pool": "^3.9.0" + }, + "peerDependencies": { + "@themost/common": "^2", + "@themost/events": "^1", + "@themost/query": "^2" + } + }, "node_modules/@themost/promise-sequence": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@themost/promise-sequence/-/promise-sequence-1.0.1.tgz", @@ -7468,6 +7484,16 @@ "node": "^12.13.0 || ^14.15.0 || >=16.0.0" } }, + "node_modules/generic-pool": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz", + "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 4" + } + }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -16233,6 +16259,15 @@ "integrity": "sha512-D/i8ONz7dgzWOa/SCoodPCg/yb5C8UQaB5T0Ob2yauLtCHR+OVbrgBnAtqQxxSM0w6cWrvPEycQTqO1Ldn4vbg==", "dev": true }, + "@themost/pool": { + "version": "2.10.1", + "resolved": "https://registry.npmjs.org/@themost/pool/-/pool-2.10.1.tgz", + "integrity": "sha512-KVn+Sh+b904a3DzWwVeilpBh3p4w/7ZpHXh1oY1ZyX4VLPPStrvA/rAX/Z49yYlRWWAiPbqtoY1jFhhBGu72Kg==", + "dev": true, + "requires": { + "generic-pool": "^3.9.0" + } + }, "@themost/promise-sequence": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/@themost/promise-sequence/-/promise-sequence-1.0.1.tgz", @@ -17951,6 +17986,12 @@ "wide-align": "^1.1.5" } }, + "generic-pool": { + "version": "3.9.0", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.9.0.tgz", + "integrity": "sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==", + "dev": true + }, "gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", diff --git a/package.json b/package.json index fee6504..131dba9 100644 --- a/package.json +++ b/package.json @@ -44,6 +44,7 @@ "@themost/common": "^2.5.11", "@themost/json-logger": "^1.1.0", "@themost/peers": "^1.0.2", + "@themost/pool": "^2.10.1", "@themost/query": "^2.6.73", "@themost/sqlite": "^2.8.4", "@themost/xml": "^2.5.2", diff --git a/platform-server/CacheEntry.js b/platform-server/CacheEntry.js new file mode 100644 index 0000000..2acedea --- /dev/null +++ b/platform-server/CacheEntry.js @@ -0,0 +1,151 @@ +/* eslint-disable quotes */ +const CacheEntrySchema = { + "$schema": "https://themost-framework.github.io/themost/models/2018/2/schema.json", + "name": "CacheEntry", + "version": "2.0.0", + "abstract": false, + "hidden": true, + "caching": "none", + "fields": [ + { + "name": "id", + "description": "A unique identifier for the current cache entry.", + "type": "Guid", + "nullable": true, + "readonly": false, + "editable": true, + "size": 36, + "primary": true + }, + { + "name": "path", + "description": "A string that represents the path of the current cache entry.", + "type": "Text", + "nullable": false, + "readonly": false, + "editable": true, + "size": 1024, + "indexed": true + }, + { + "name": "headers", + "description": "A key-value pair string that represents the request headers being passed by the process.", + "type": "Text", + "nullable": true, + "readonly": false, + "editable": true, + "size": 1024 + }, + { + "name": "doomed", + "description": "A boolean value that indicates whether the current cache entry is doomed.", + "type": "Boolean", + "nullable": true, + "readonly": false, + "editable": true, + "indexed": true + }, + { + "name": "contentEncoding", + "description": "A string that represents the content encoding of the current cache entry.", + "type": "Text", + "nullable": false, + "readonly": false, + "editable": true + }, + { + "name": "location", + "type": "Text", + "nullable": false, + "readonly": false, + "editable": true, + "size": 24 + }, + { + "name": "params", + "description": "A string that represents the route parameters being passed by the process.", + "type": "Text", + "nullable": true, + "readonly": false, + "editable": true, + "size": 1024 + }, + { + "name": "customParams", + "description": "A string that represents the custom parameters being passed by the process.", + "type": "Text", + "nullable": true, + "readonly": false, + "editable": true, + "size": 1024 + }, + { + "name": "duration", + "description": "An integer that represents the absolute duration of the current cache entry.", + "type": "Integer", + "readonly": false, + "editable": false + }, + { + "name": "createdAt", + "description": "A date that represents the creation date of the current cache entry.", + "type": "DateTime", + "nullable": false, + "readonly": false, + "editable": false + }, + { + "name": "expiredAt", + "description": "A date that represents the expiration date of the current cache entry.", + "type": "DateTime", + "readonly": false, + "editable": true + }, + { + "name": "modifiedAt", + "description": "A date that represents the last modification date of the current cache entry.", + "readonly": false, + "editable": true + }, + { + "name": "entityTag", + "description": "A string that represents the generated entity tag of the current cache entry.", + "type": "Text", + "readonly": false, + "editable": true + }, + { + "name": "content", + "description": "A string that represents the content of the current cache entry.", + "type": "Json" + } + ], + "constraints": [ + { + "type": "unique", + "fields": [ + "path", + "location", + "contentEncoding", + "headers", + "params", + "customParams" + ] + } + ], + "privileges": [ + { + "mask": 15, + "type": "global" + }, + { + "mask": 15, + "type": "global", + "account": "Administrators" + } + ] +} + +module.exports = { + CacheEntrySchema +}; \ No newline at end of file diff --git a/platform-server/MemoryCacheStrategy.d.ts b/platform-server/MemoryCacheStrategy.d.ts new file mode 100644 index 0000000..5b36dc7 --- /dev/null +++ b/platform-server/MemoryCacheStrategy.d.ts @@ -0,0 +1,9 @@ +import { DataCacheStrategy } from '../data-cache'; + +export declare class MemoryCacheStrategy extends DataCacheStrategy { + add(key: string, value: any, absoluteExpiration?: number): Promise; + remove(key: string): Promise; + clear(): Promise; + get(key: string): Promise; + getOrDefault(key: string, getFunc: () => Promise, absoluteExpiration?: number): Promise; +} \ No newline at end of file diff --git a/platform-server/MemoryCacheStrategy.js b/platform-server/MemoryCacheStrategy.js new file mode 100644 index 0000000..d3952a2 --- /dev/null +++ b/platform-server/MemoryCacheStrategy.js @@ -0,0 +1,181 @@ +/* eslint-disable node/no-unpublished-require */ +const { DataApplication } = require('../data-application'); +const { DataCacheStrategy } = require('../data-cache'); +const { SqliteAdapter } = require('@themost/sqlite'); +const { GenericPoolAdapter, createInstance } = require('@themost/pool'); +const { DataConfigurationStrategy, SchemaLoaderStrategy } = require('../data-configuration'); +const path = require('path'); +const fs = require('fs'); +const { Guid, TraceUtils } = require('@themost/common'); +const { CacheEntrySchema } = require('./CacheEntry'); + +class MemoryCacheApplication extends DataApplication { + constructor() { + super(path.resolve(process.cwd(), '.cache')); + const config = this.configuration.getStrategy(DataConfigurationStrategy); + Object.assign(config.adapterTypes, { + sqlite: { + invariantName: 'sqlite', + type: SqliteAdapter, + createInstance: (options) => { + return new SqliteAdapter(options); + } + }, + pool: { + invariantName: 'pool', + type: GenericPoolAdapter, + createInstance: createInstance + } + }); + config.adapters.push( + { + name: 'cache+pool', + invariantName: 'pool', + default: true, + options: { + adapter: 'cache', + max: 25, + min: 2 + } + },{ + name: 'cache', + invariantName: 'sqlite', + options: { + database: path.resolve(process.cwd(), '.cache', 'cache.db') + } + } + ); + const schema = this.configuration.getStrategy(SchemaLoaderStrategy); + schema.setModelDefinition(CacheEntrySchema); + const executionPath = this.getConfiguration().getExecutionPath(); + TraceUtils.debug('Validating cache service path: ' + executionPath); + void fs.stat(executionPath, (err) => { + if (err) { + if (err.code === 'ENOENT') { + TraceUtils.debug('Creating cache service path: ' + executionPath); + void fs.mkdir(executionPath, (err) => { + if (err) { + TraceUtils.error(err); + return; + } + TraceUtils.debug('Cache service path created successfully.'); + }); + } else { + TraceUtils.error(err); + } + } + }); + } +} + +class MemoryCacheStrategy extends DataCacheStrategy { + + constructor(config) { + super(config); + this.cache = new MemoryCacheApplication(); + } + + async get(key) { + const context = this.cache.createContext(); + try { + const entry = await context.model('CacheEntry').asQueryable().where((x, key) => { + return x.path === key && x.location === 'server'; + }, key).getItem(); + if (entry && entry.doomed) { + return; + } + if (entry && entry.content) { + return entry.content; + } + return; + } finally { + await context.finalizeAsync(); + } + } + + async add(key, value, absoluteExpiration) { + const context = this.cache.createContext(); + const CacheEntries = context.model('CacheEntry'); + try { + const entry = await CacheEntries.asQueryable().where((x, key) => { + return x.path === key && + x.location === 'server' && + x.contentEncoding === 'application/json'; + }, key).select( + ({id}) => ({id}) + ).getItem(); + const id = entry && entry.id; + await CacheEntries.upsert({ + id: id || Guid.newGuid().toString(), + path: key, + location: 'server', + content: value, + contentEncoding: 'application/json', + createdAt: new Date(), + modifiedAt: new Date(), + expiredAt: absoluteExpiration ? new Date(Date.now() + (absoluteExpiration || 0)) : null, + doomed: false + }); + return value; + } finally { + await context.finalizeAsync(); + } + } + async remove(key) { + const context = this.cache.createContext(); + const CacheEntries = context.model('CacheEntry'); + try { + await CacheEntries.remove({ + path: key + }); + } finally { + await context.finalizeAsync(); + } + } + + async clear() { + throw new Error('Method not implemented.'); + } + + async getOrDefault(key, getFunc, absoluteExpiration) { + const context = this.cache.createContext(); + const CacheEntries = context.model('CacheEntry'); + try { + const exists = await CacheEntries.asQueryable().where((x, key) => { + return x.path === key && + x.location === 'server' && + x.contentEncoding === 'application/json' && + x.doomed === false; + }, key).count(); + if (exists) { + return CacheEntries.asQueryable().where((x) => { + return x.path === key && x.doomed === false; + }).select(({content}) => ({ + content + })).value().then((result) => { + if (result) { + return JSON.parse(result.content); + } + return null; + }); + } + const result = await getFunc(); + await CacheEntries.insert({ + id: Guid.newGuid().toString(), + path: key, + location: 'server', + content: result, + createdAt: new Date(), + modifiedAt: new Date(), + contentEncoding: 'application/json', + expiredAt: absoluteExpiration ? new Date(Date.now() + (absoluteExpiration || 0)) : null, + doomed: false + }); + return result; + } finally { + await context.finalizeAsync(); + } + } +} + +module.exports = { MemoryCacheStrategy }; \ No newline at end of file diff --git a/platform-server/index.d.ts b/platform-server/index.d.ts new file mode 100644 index 0000000..09e3b5c --- /dev/null +++ b/platform-server/index.d.ts @@ -0,0 +1 @@ +export * from './MemoryCacheStrategy' \ No newline at end of file diff --git a/platform-server/index.js b/platform-server/index.js new file mode 100644 index 0000000..9cc5ce0 --- /dev/null +++ b/platform-server/index.js @@ -0,0 +1,4 @@ +const { MemoryCacheStrategy } = require('./MemoryCacheStrategy'); +module.exports = { + MemoryCacheStrategy +} \ No newline at end of file diff --git a/platform-server/package.json b/platform-server/package.json new file mode 100644 index 0000000..f41d759 --- /dev/null +++ b/platform-server/package.json @@ -0,0 +1,10 @@ +{ + "name": "themost-data-platform-server", + "private": true, + "peerDependencies": { + "@themost/query": "^2.6.73", + "@themost/sqlite": "^2.8.4", + "@themost/pool": "^2.10.1", + "@themost/common": "^2.5.11" + } +} \ No newline at end of file diff --git a/spec/MemoryCacheEntry.spec.ts b/spec/MemoryCacheEntry.spec.ts new file mode 100644 index 0000000..2ebffdb --- /dev/null +++ b/spec/MemoryCacheEntry.spec.ts @@ -0,0 +1,37 @@ +import {TestApplication} from './TestApplication'; +import {DataContext} from '../types'; +import {resolve} from 'path'; +import { DataCacheStrategy } from '../data-cache'; +import { MemoryCacheStrategy } from '@themost/data/platform-server'; +import { SchemaLoaderStrategy } from '../data-configuration'; + +describe('MemoryCacheEntry', () => { + let app: TestApplication; + let context: DataContext; + beforeAll((done) => { + app = new TestApplication(resolve(__dirname, 'test2')); + app.getConfiguration().useStrategy( + DataCacheStrategy, + MemoryCacheStrategy + ) + context = app.createContext(); + return done(); + }); + afterAll(async () => { + await context.finalizeAsync(); + await app.finalize(); + }); + + it('should create memory cache entry', async () => { + const schema = context.getConfiguration().getStrategy(SchemaLoaderStrategy); + const model = schema.getModelDefinition('Group'); + model.caching = 'always'; + schema.setModelDefinition(model); + const Groups = context.model('Group'); + let items = await Groups.getItems(); + expect(items).toBeTruthy(); + for (let i = 0; i < 10; i++) { + items = await Groups.getItems(); + } + }); +}); \ No newline at end of file diff --git a/spec/MemoryCacheStrategy.ts b/spec/MemoryCacheStrategy.ts deleted file mode 100644 index a08c634..0000000 --- a/spec/MemoryCacheStrategy.ts +++ /dev/null @@ -1,81 +0,0 @@ -import { DataApplication } from "../data-application"; -import { DataCacheStrategy } from "../data-cache"; -import { SqliteAdapter } from "@themost/sqlite"; -import { DataConfigurationStrategy, SchemaLoaderStrategy } from "../data-configuration"; -import path from 'path'; -import { ConfigurationBase, DataModelProperties } from "@themost/common"; - -class MemoryCacheApplication extends DataApplication { - constructor() { - super(path.resolve(process.cwd(), '.cache')); - const config = this.configuration.getStrategy(DataConfigurationStrategy); - config.adapterTypes.set('sqlite', { - name: 'sqlite', - invariantName: 'sqlite', - type: SqliteAdapter - }); - config.adapters.push({ - name: 'cache', - invariantName: 'sqlite', - options: { - database: ':memory:' - } - }); - const schema = this.getService(SchemaLoaderStrategy); - schema.setModelDefinition({ - name: 'MemoryCache', - title: 'Memory Cache', - caching: 'none', - version: '0.0.0', - fields: [ - { - name: 'id', - type: 'Text', - primary: true, - nullable: false - }, - { - name: 'additionalType', - type: 'Text' - }, - { - name: 'value', - type: 'Text' - }, - { - name: 'expiresAt', - type: 'DateTime' - } - ] - } as DataModelProperties) - } -} - -class MemoryCacheStrategy extends DataCacheStrategy { - - protected cache: MemoryCacheApplication; - - constructor(config: ConfigurationBase) { - super(config); - this.cache = new MemoryCacheApplication(); - } - - get(key: string): Promise { - throw new Error("Method not implemented."); - } - - add(key: string, value: any, absoluteExpiration?: number): Promise { - throw new Error("Method not implemented."); - } - remove(key: string): Promise { - throw new Error("Method not implemented."); - } - clear(): Promise { - throw new Error("Method not implemented."); - } - getOrDefault(key: string, getFunc: () => Promise, absoluteExpiration?: number): Promise { - throw new Error("Method not implemented."); - } -} - -export { MemoryCacheStrategy }; \ No newline at end of file diff --git a/tsconfig.json b/tsconfig.json index 23cdefa..2b892af 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -17,6 +17,9 @@ "dom" ], "paths": { + "@themost/data/platform-server": [ + "./platform-server/index" + ], "@themost/data": [ "./index" ] From 01d1e2a3ec6b1627cd5c79a2702ab0ea13b4e471 Mon Sep 17 00:00:00 2001 From: Kyriakos Barbounakis Date: Wed, 15 Jan 2025 14:24:19 +0200 Subject: [PATCH 3/4] generate cache entry id --- package-lock.json | 12 +++++ package.json | 1 + platform-server/CacheEntry.js | 3 +- platform-server/MemoryCacheStrategy.js | 75 +++++++++++++------------- platform-server/package.json | 3 +- 5 files changed, 56 insertions(+), 38 deletions(-) diff --git a/package-lock.json b/package-lock.json index 8d7b705..013f378 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "@themost/promise-sequence": "^1.0.1", "ajv": "^8.17.1", "async": "^2.6.4", + "crypto-js": "^4.2.0", "lodash": "^4.17.21", "node-cache": "^5.1.2", "pluralize": "^7.0.0", @@ -6423,6 +6424,12 @@ "node": ">= 8" } }, + "node_modules/crypto-js": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-4.2.0.tgz", + "integrity": "sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==", + "license": "MIT" + }, "node_modules/date-and-time": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/date-and-time/-/date-and-time-3.1.1.tgz", @@ -17192,6 +17199,11 @@ "which": "^2.0.1" } }, + "crypto-js": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/crypto-js/-/crypto-js-4.2.0.tgz", + "integrity": "sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==" + }, "date-and-time": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/date-and-time/-/date-and-time-3.1.1.tgz", diff --git a/package.json b/package.json index 131dba9..b324931 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "@themost/promise-sequence": "^1.0.1", "ajv": "^8.17.1", "async": "^2.6.4", + "crypto-js": "^4.2.0", "lodash": "^4.17.21", "node-cache": "^5.1.2", "pluralize": "^7.0.0", diff --git a/platform-server/CacheEntry.js b/platform-server/CacheEntry.js index 2acedea..696b070 100644 --- a/platform-server/CacheEntry.js +++ b/platform-server/CacheEntry.js @@ -59,7 +59,8 @@ const CacheEntrySchema = { "nullable": false, "readonly": false, "editable": true, - "size": 24 + "size": 24, + "indexed": true }, { "name": "params", diff --git a/platform-server/MemoryCacheStrategy.js b/platform-server/MemoryCacheStrategy.js index d3952a2..a2a68c6 100644 --- a/platform-server/MemoryCacheStrategy.js +++ b/platform-server/MemoryCacheStrategy.js @@ -8,6 +8,20 @@ const path = require('path'); const fs = require('fs'); const { Guid, TraceUtils } = require('@themost/common'); const { CacheEntrySchema } = require('./CacheEntry'); +const MD5 = require('crypto-js/md5'); + +if (typeof Guid.from !== 'function') { + Guid.from = function(value) { + var str = MD5(value).toString(); + return new Guid([ + str.substring(0, 8), + str.substring(8, 12), + str.substring(12, 16), + str.substring(16, 20), + str.substring(20, 32) + ].join('-')); + } +} class MemoryCacheApplication extends DataApplication { constructor() { @@ -97,25 +111,29 @@ class MemoryCacheStrategy extends DataCacheStrategy { const context = this.cache.createContext(); const CacheEntries = context.model('CacheEntry'); try { - const entry = await CacheEntries.asQueryable().where((x, key) => { - return x.path === key && - x.location === 'server' && - x.contentEncoding === 'application/json'; - }, key).select( - ({id}) => ({id}) - ).getItem(); - const id = entry && entry.id; - await CacheEntries.upsert({ - id: id || Guid.newGuid().toString(), + // create uuid from unique constraint attributes + // (avoid checking if exists) + const entry = { path: key, location: 'server', - content: value, contentEncoding: 'application/json', + headers: null, + params: null, + customParams: null + } + // get id + const id = Guid.from(entry).toString(); + // assign extra properties + Object.assign(entry, { + id: id, + content: value, createdAt: new Date(), modifiedAt: new Date(), expiredAt: absoluteExpiration ? new Date(Date.now() + (absoluteExpiration || 0)) : null, doomed: false }); + // insert or update cache entry + await CacheEntries.upsert(entry); return value; } finally { await context.finalizeAsync(); @@ -141,36 +159,21 @@ class MemoryCacheStrategy extends DataCacheStrategy { const context = this.cache.createContext(); const CacheEntries = context.model('CacheEntry'); try { - const exists = await CacheEntries.asQueryable().where((x, key) => { + const entry = await CacheEntries.asQueryable().where((x, key) => { return x.path === key && x.location === 'server' && - x.contentEncoding === 'application/json' && x.doomed === false; - }, key).count(); - if (exists) { - return CacheEntries.asQueryable().where((x) => { - return x.path === key && x.doomed === false; - }).select(({content}) => ({ - content - })).value().then((result) => { - if (result) { - return JSON.parse(result.content); - } - return null; - }); + }, key).select(({content, contentEncoding}) => ({ + content, contentEncoding + })).getItem(); + if (entry && typeof entry.content !== 'undefined') { + return entry.content; } + // get value from function const result = await getFunc(); - await CacheEntries.insert({ - id: Guid.newGuid().toString(), - path: key, - location: 'server', - content: result, - createdAt: new Date(), - modifiedAt: new Date(), - contentEncoding: 'application/json', - expiredAt: absoluteExpiration ? new Date(Date.now() + (absoluteExpiration || 0)) : null, - doomed: false - }); + // add value to cache + await this.add(key, result, absoluteExpiration); + // return value return result; } finally { await context.finalizeAsync(); diff --git a/platform-server/package.json b/platform-server/package.json index f41d759..eab6663 100644 --- a/platform-server/package.json +++ b/platform-server/package.json @@ -5,6 +5,7 @@ "@themost/query": "^2.6.73", "@themost/sqlite": "^2.8.4", "@themost/pool": "^2.10.1", - "@themost/common": "^2.5.11" + "@themost/common": "^2.5.11", + "crypto-js": "^4.2.0" } } \ No newline at end of file From ebf48971a3aea5a16cc776d947365c584504d416 Mon Sep 17 00:00:00 2001 From: Kyriakos Barbounakis Date: Wed, 15 Jan 2025 23:29:04 +0200 Subject: [PATCH 4/4] validate cache strategy --- platform-server/MemoryCacheStrategy.js | 105 ++++++++++++++++++------- spec/MemoryCacheEntry.spec.ts | 12 +++ 2 files changed, 88 insertions(+), 29 deletions(-) diff --git a/platform-server/MemoryCacheStrategy.js b/platform-server/MemoryCacheStrategy.js index a2a68c6..3c759d2 100644 --- a/platform-server/MemoryCacheStrategy.js +++ b/platform-server/MemoryCacheStrategy.js @@ -9,6 +9,9 @@ const fs = require('fs'); const { Guid, TraceUtils } = require('@themost/common'); const { CacheEntrySchema } = require('./CacheEntry'); const MD5 = require('crypto-js/md5'); +const { QueryExpression } = require('@themost/query'); + +const CACHE_ABSOLUTE_EXPIRATION = 1200; if (typeof Guid.from !== 'function') { Guid.from = function(value) { @@ -79,6 +82,12 @@ class MemoryCacheApplication extends DataApplication { } } }); + const absoluteExpiration = this.configuration.getSourceAt('settings/cache/absoluteExpiration'); + if (typeof absoluteExpiration === 'number' && absoluteExpiration >= 0) { + this.absoluteExpiration = absoluteExpiration; + } else { + this.absoluteExpiration = CACHE_ABSOLUTE_EXPIRATION; + } } } @@ -86,18 +95,42 @@ class MemoryCacheStrategy extends DataCacheStrategy { constructor(config) { super(config); - this.cache = new MemoryCacheApplication(); + this.rawCache = new MemoryCacheApplication(); } + /** + * Gets a key value pair from cache + * @param {*} key + * @returns Promise<*> + */ async get(key) { - const context = this.cache.createContext(); + const context = this.rawCache.createContext(); try { const entry = await context.model('CacheEntry').asQueryable().where((x, key) => { return x.path === key && x.location === 'server'; }, key).getItem(); + const {sourceAdapter: CacheEntries} = context.model('CacheEntry'); if (entry && entry.doomed) { + // execute ad-hoc query + // remove doomed entry + await context.db.executeAsync( + new QueryExpression().delete(CacheEntries).where((x, id) => { + return x.id === id; + }, entry.id) + ); return; } + if (entry && entry.expiredAt && entry.expiredAt < new Date()) { + // execute ad-hoc query + // set doomed to true + await context.db.executeAsync( + new QueryExpression().update(CacheEntries).set({ + doomed: true + }).where((x, id) => { + return x.id === id; + }, entry.id) + ); + } if (entry && entry.content) { return entry.content; } @@ -107,8 +140,15 @@ class MemoryCacheStrategy extends DataCacheStrategy { } } + /** + * Sets a key value pair in cache. + * @param {*} key - The key to be cached + * @param {*} value - The value to be cached + * @param {number=} absoluteExpiration - The expiration time in seconds + * @returns {Promise<*>} + */ async add(key, value, absoluteExpiration) { - const context = this.cache.createContext(); + const context = this.rawCache.createContext(); const CacheEntries = context.model('CacheEntry'); try { // create uuid from unique constraint attributes @@ -129,7 +169,7 @@ class MemoryCacheStrategy extends DataCacheStrategy { content: value, createdAt: new Date(), modifiedAt: new Date(), - expiredAt: absoluteExpiration ? new Date(Date.now() + (absoluteExpiration || 0)) : null, + expiredAt: absoluteExpiration ? new Date(Date.now() + ((absoluteExpiration || 0) * 1000)) : null, doomed: false }); // insert or update cache entry @@ -139,13 +179,22 @@ class MemoryCacheStrategy extends DataCacheStrategy { await context.finalizeAsync(); } } + + /** + * Removes a key from cache + * @param {*} key + */ async remove(key) { - const context = this.cache.createContext(); - const CacheEntries = context.model('CacheEntry'); + const context = this.rawCache.createContext(); + const {sourceAdapter: CacheEntries} = context.model('CacheEntry'); try { - await CacheEntries.remove({ - path: key - }); + // remove using an ad-hoc query to support wildcard characters + const searchPath = key.replace(/\*/g, '%'); + await context.db.executeAsync( + new QueryExpression().delete(CacheEntries).where((x, search) => { + return x.path.includes(search) === true && x.location === 'server'; + }, searchPath) + ); } finally { await context.finalizeAsync(); } @@ -155,29 +204,27 @@ class MemoryCacheStrategy extends DataCacheStrategy { throw new Error('Method not implemented.'); } + /** + * Gets a key value pair from cache or invokes the given function and returns the value before caching it. + * @param {*} key + * @param {function():Promise<*>} getFunc The function to be invoked if the key is not found in cache + * @param {number=} absoluteExpiration The expiration time in seconds + * @returns + */ async getOrDefault(key, getFunc, absoluteExpiration) { - const context = this.cache.createContext(); - const CacheEntries = context.model('CacheEntry'); - try { - const entry = await CacheEntries.asQueryable().where((x, key) => { - return x.path === key && - x.location === 'server' && - x.doomed === false; - }, key).select(({content, contentEncoding}) => ({ - content, contentEncoding - })).getItem(); - if (entry && typeof entry.content !== 'undefined') { - return entry.content; - } - // get value from function - const result = await getFunc(); - // add value to cache - await this.add(key, result, absoluteExpiration); + // try to get entry from cache + const value = await this.get(key); + // if entry exists + if (typeof value !== 'undefined') { // return value - return result; - } finally { - await context.finalizeAsync(); + return value; } + // otherwise, get value by invoking function + const result = await getFunc(); + // add value to cache + await this.add(key, typeof result === 'undefined' ? null : result, absoluteExpiration); + // return value + return result; } } diff --git a/spec/MemoryCacheEntry.spec.ts b/spec/MemoryCacheEntry.spec.ts index 2ebffdb..3731a26 100644 --- a/spec/MemoryCacheEntry.spec.ts +++ b/spec/MemoryCacheEntry.spec.ts @@ -34,4 +34,16 @@ describe('MemoryCacheEntry', () => { items = await Groups.getItems(); } }); + + it('should delete memory cache entry', async () => { + const schema = context.getConfiguration().getStrategy(SchemaLoaderStrategy); + const model = schema.getModelDefinition('Group'); + model.caching = 'always'; + schema.setModelDefinition(model); + const Groups = context.model('Group'); + await Groups.getItems(); + const cache: any = app.getConfiguration().getStrategy(DataCacheStrategy); + await cache.remove('/Group/*'); + + }); }); \ No newline at end of file