中间计量表，缓存查询数据

app/controller/report_controller.js

@@ -282,8 +282,6 @@ async function getReportData(ctx, params, filters) {
     const rst = {};
     const runnableRst = [];
     const runnableKey = []; // 这个配合runnableRst用，未来考虑并行查询优化
-    const mem_tz_runnableRst = []; // 内存表，中间计量台账清单数据，因有主从关系，但却又只算一次，所以上面的并行机制无法满足需求，所以得走另外的分支处理
-    const mem_tz_runnableKey = []; // 配合内存表用
     // console.log('params');
     // console.log(params);
     for (const filter of filters) {
@@ -317,18 +315,6 @@ async function getReportData(ctx, params, filters) {
                     runnableRst.push(ctx.service.stagePay.getAuditorStageData(params.stage_id, params.stage_times, params.stage_order));
                     runnableKey.push('stage_pay');
                     break;
-                case 'mem_stage_im_tz':
-                    if (mem_tz_runnableRst.length === 0) {
-                        mem_tz_runnableRst.push(ctx.service.reportMemory.getStageImTzNoReturn(params.tender_id, params.stage_id));
-                    }
-                    mem_tz_runnableKey.push('mem_stage_im_tz');
-                    break;
-                case 'mem_stage_im_tz_bills':
-                    if (mem_tz_runnableRst.length === 0) {
-                        mem_tz_runnableRst.push(ctx.service.reportMemory.getStageImTzNoReturn(params.tender_id, params.stage_id));
-                    }
-                    mem_tz_runnableKey.push('mem_stage_im_tz_bills');
-                    break;
                 case 'mem_stage_im_zl':
                     runnableRst.push(ctx.service.reportMemory.getStageImZlData(params.tender_id, params.stage_id));
                     runnableKey.push('mem_stage_im_zl');
@@ -348,10 +334,16 @@ async function getReportData(ctx, params, filters) {
         // if (rst[runnableKey[idx]] instanceof Array) console.log('is Array')
         // else console.log('is not Array');
     }
-    if (mem_tz_runnableRst.length > 0) {
-        await mem_tz_runnableRst[0];
-        for (const key of mem_tz_runnableKey) {
-            rst[key] = ctx.service.reportMemory.getStageImTzDataDirectlyByKey(key);
+    for (const filter of filters) {
+        switch (filter) {
+            case 'mem_stage_im_tz':
+                rst[filter] = await ctx.service.reportMemory.getStageImTzData(params.tender_id, params.stage_id);
+                break;
+            case 'mem_stage_im_tz_bills':
+                rst[filter] = await ctx.service.reportMemory.getStageImTzBillsData(params.tender_id, params.stage_id);
+                break;
+            default:
+                break;
         }
     }
     return rst;

app/controller/stage_controller.js

@@ -351,6 +351,7 @@ module.exports = app => {
                         responseData.data = await ctx.service.stageBills.updateStageBillsCalcType(data.bills.calcType);                    }
                 }
                 await ctx.service.stage.updateCheckCalcFlag(ctx.stage.id, true);
+                await ctx.service.stage.updateCacheTime(ctx.stage.id);
                 ctx.body = responseData;
             } catch (err) {
                 this.log(err);
@@ -404,6 +405,7 @@ module.exports = app => {
                         ctx.stage.id, data.target.bills.id, '-1');
                 }
                 await ctx.service.stage.updateCheckCalcFlag(ctx.stage.id, true);
+                await ctx.service.stage.updateCacheTime(ctx.stage.id);
                 ctx.body = {err: 0, msg: '', data: result};
             } catch(err) {
                 this.log(err);
@@ -458,6 +460,7 @@ module.exports = app => {
 
                 const data = JSON.parse(ctx.request.body.data);
                 await ctx.service.stage.buildDetailData(ctx.tender.id, ctx.stage.order, data);
+                await ctx.service.stage.updateCacheTime(ctx.stage.id);
 
                 ctx.body = {err: 0, msg: '', data: null};
             } catch (err) {
@@ -522,6 +525,7 @@ module.exports = app => {
 
                 const data = JSON.parse(ctx.request.body.data);
                 await ctx.service.stage.update(data, { id: ctx.stage.id });
+                await ctx.service.stage.updateCacheTime(ctx.stage.id);
 
                 ctx.body = {err: 0, msg: '', data: this.ctx.stage};
             } catch (err) {
@@ -546,6 +550,7 @@ module.exports = app => {
                 } else {
                     responseData.data = await ctx.service.stageDetail.saveDetailData(data);
                 }
+                await ctx.service.stage.updateCacheTime(ctx.stage.id);
                 ctx.body = responseData;
             } catch (err) {
                 this.log(err);
@@ -603,6 +608,7 @@ module.exports = app => {
                 }
                 await this.ctx.service.stageDetail.saveDetailData(data);
                 const imData = await ctx.service.stageDetail.getLastestImStageData(this.ctx.tender.id, this.ctx.stage.id, data.lid, data.uuid);
+                await ctx.service.stage.updateCacheTime(ctx.stage.id);
                 const responseData = {err: 0, msg: '', data: imData};
                 ctx.body = responseData;
             } catch (err) {

app/extend/helper.js

@@ -504,7 +504,6 @@ module.exports = {
      * @returns {Promise<void>}
      */
     async saveBufferFile(buffer, fileName) {
-        console.log(fileName);
         // 检查文件夹是否存在，不存在则直接创建文件夹
         const pathName = path.dirname(fileName);
         if (!fs.existsSync(pathName)) {

app/service/report_memory.js

@@ -8,13 +8,18 @@
  * @version
  */
 
-const Service = require('egg').Service;
 const StageIm = require('../lib/stage_im');
 const imType = require('../const/tender').imType;
 const audit = require('../const/audit');
+// const path = require('path');
+// const fs = require('fs');
+
+const stageImTz = 'mem_stage_im_tz';
+const stageImTzBills = 'mem_stage_im_tz_bills';
+const stageImZl = 'mem_stage_im_zl';
 
 module.exports = app => {
-    class ReportMemory extends Service {
+    class ReportMemory extends app.BaseService {
 
         /**
          * 构造函数
@@ -24,69 +29,117 @@ module.exports = app => {
          */
         constructor(ctx) {
             super(ctx);
-            this.db = this.app.mysql;
-            this.cache = this.app.redis;
-            this._ = this.app._;
+            this.tableName = 'report_memory';
             // 需要缓存的数据
             this.stageImData = null;
         }
 
         async _checkTender(tid) {
+            if (this.ctx.tender) return;
             const tender = await this.ctx.service.tender.getTender(tid);
             tender.info = await this.ctx.service.tenderInfo.getTenderInfo(tid);
             this.ctx.tender = tender;
         }
 
         async _checkStage(sid) {
-            const status = audit.stage.status;
-            const stage = await this.ctx.service.stage.getDataById(sid);
-            stage.auditors = await this.ctx.service.stageAudit.getAuditors(stage.id, stage.times);
-            stage.curAuditor = await this.ctx.service.stageAudit.getCurAuditor(stage.id, stage.times);
-
-            const accountId = this.ctx.session.sessionUser.accountId, auditorIds = this._.map(stage.auditors, 'aid'), shareIds = [];
-            if (accountId === stage.user_id) { // 原报
-                if (stage.curAuditor) {
-                    stage.readOnly = stage.curAuditor.aid !== accountId;
-                } else {
-                    stage.readOnly = stage.status !== status.uncheck && stage.status !== status.checkNo;
-                }
-                stage.curTimes = stage.times;
-                if (stage.status === status.uncheck || stage.status === status.checkNo) {
-                    stage.curOrder = 0;
-                } else if (stage.status === status.checked) {
-                    stage.curOrder = this._.max(this._.map(stage.auditors, 'order'));
-                } else {
-                    stage.curOrder = stage.curAuditor.aid === accountId ? stage.curAuditor.order : stage.curAuditor.order - 1;
-                }
-            } else if (auditorIds.indexOf(accountId) !== -1) { // 审批人
-                if (stage.status === status.uncheck) {
-                    throw '您无权查看该数据';
-                }
-                stage.curTimes = stage.status === status.checkNo ? stage.times - 1 : stage.times;
-                if (stage.status === status.checked) {
-                    stage.curOrder = this._.max(this._.map(stage.auditors, 'order'));
-                } else if (stage.status === status.checkNo) {
-                    const audit = await this.service.stageAudit.getDataByCondition({
-                        sid: stage.id, times: stage.times - 1, status: status.checkNo
-                    });
-                    stage.curOrder = audit.order;
-                } else {
-                    stage.curOrder = accountId === stage.curAuditor.aid ? stage.curAuditor.order : stage.curAuditor.order - 1;
-                }
-            } else if (shareIds.indexOf(accountId) !== -1) { // 分享人
-                if (stage.status === status.uncheck) {
-                    throw '您无权查看该数据';
+            if (!this.ctx.stage) {
+                const status = audit.stage.status;
+                const stage = await this.ctx.service.stage.getDataById(sid);
+                stage.auditors = await this.ctx.service.stageAudit.getAuditors(stage.id, stage.times);
+                stage.curAuditor = await this.ctx.service.stageAudit.getCurAuditor(stage.id, stage.times);
+
+                const accountId = this.ctx.session.sessionUser.accountId, auditorIds = this._.map(stage.auditors, 'aid'), shareIds = [];
+                if (accountId === stage.user_id) { // 原报
+                    if (stage.curAuditor) {
+                        stage.readOnly = stage.curAuditor.aid !== accountId;
+                    } else {
+                        stage.readOnly = stage.status !== status.uncheck && stage.status !== status.checkNo;
+                    }
+                    stage.curTimes = stage.times;
+                    if (stage.status === status.uncheck || stage.status === status.checkNo) {
+                        stage.curOrder = 0;
+                    } else if (stage.status === status.checked) {
+                        stage.curOrder = this._.max(this._.map(stage.auditors, 'order'));
+                    } else {
+                        stage.curOrder = stage.curAuditor.aid === accountId ? stage.curAuditor.order : stage.curAuditor.order - 1;
+                    }
+                } else if (auditorIds.indexOf(accountId) !== -1) { // 审批人
+                    if (stage.status === status.uncheck) {
+                        throw '您无权查看该数据';
+                    }
+                    stage.curTimes = stage.status === status.checkNo ? stage.times - 1 : stage.times;
+                    if (stage.status === status.checked) {
+                        stage.curOrder = this._.max(this._.map(stage.auditors, 'order'));
+                    } else if (stage.status === status.checkNo) {
+                        const audit = await this.service.stageAudit.getDataByCondition({
+                            sid: stage.id, times: stage.times - 1, status: status.checkNo
+                        });
+                        stage.curOrder = audit.order;
+                    } else {
+                        stage.curOrder = accountId === stage.curAuditor.aid ? stage.curAuditor.order : stage.curAuditor.order - 1;
+                    }
+                } else if (shareIds.indexOf(accountId) !== -1) { // 分享人
+                    if (stage.status === status.uncheck) {
+                        throw '您无权查看该数据';
+                    }
+                    stage.curTimes = stage.status === status.checkNo ? stage.times - 1 : stage.times;
+                    stage.curOrder = stage.status === status.checked ? this._.max(this._.map(stage.auditors, 'order')) : stage.curAuditor.order - 1;
                 }
-                stage.curTimes = stage.status === status.checkNo ? stage.times - 1 : stage.times;
-                stage.curOrder = stage.status === status.checked ? this._.max(this._.map(stage.auditors, 'order')) : stage.curAuditor.order - 1;
+
+                this.ctx.stage = stage;
+                this.ctx.stage.cacheTime = this.ctx.stage.readOnly ? (this.ctx.stage.cache_time_r).getTime(): (this.ctx.stage.cache_time_l).getTime();
+            }
+        }
+
+        // build-time: 162-384ms, redis-cache: 0-41ms, mysql + IO: 116-146ms
+        // 一定程度上算是大Value缓存，数据多了以后：
+        // 1. 达到redis内存阈值时，数据会swap到磁盘，此时将消耗IO时间
+        // 2. redis单独服务器
+        // 3. redis集群
+        async _getReportMemoryCache(name, tid, sid, time) {
+            // redis
+            const cacheKey = name + '-t' + tid + (sid ? '-s' + sid : '') + (time ? '-' + time : '');
+            const data = await this.cache.get(cacheKey);
+            if (data) {
+                return eval(data);
+            } else {
+                return null;
             }
 
-            this.ctx.stage = stage;
+            // mysql + IO
+            // const rm = await this.getDataByCondition({
+            //     tid: tid, sid: sid, name: name, time: time
+            // });
+            // if (rm && rm.file) {
+            //     const file = path.join(this.ctx.app.config.filePath, 'report', 'cache', rm.file);
+            //     if (fs.existsSync(file)) {
+            //         const data = await fs.readFileSync(file, 'utf8');
+            //         return eval(data);
+            //     } else {
+            //         return null;
+            //     }
+            // }
+        }
+
+        async _setReportMemoryCache(name, tid, sid, time, data) {
+            // redis
+            const cacheKey = name + '-t' + tid + (sid ? '-s' + sid : '') + (time ? '-' + time : '');
+            this.cache.set(cacheKey, JSON.stringify(data), 'EX', this.ctx.app.config.cacheTime);
+
+            // mysql + IO
+            // const file = path.join('report', 'cache', 'rm' + (new Date()).getTime() + '.json');
+            // await this.ctx.helper.saveBufferFile(JSON.stringify(data), path.join(this.ctx.app.config.filePath, file));
+            // const rm = await this.getDataByCondition({
+            //     tid: tid, sid: sid, name: name, time: time
+            // });
+            // if (rm) {
+            //     await this.db.update(this.tableName, {id: rm.id, file: file});
+            // } else {
+            //     await this.db.insert(this.tableName, {tid: tid, sid: sid, name: name, time: time, file: file});
+            // }
         }
 
         async _generateStageIm(tid, sid, isTz = true) {
-            await this._checkTender(tid);
-            await this._checkStage(sid);
             if (isTz && this.ctx.stage.im_type !== imType.tz.value) {
                 throw '您查看的报表跟设置不符，请查看“总量控制”的报表';
             } else if (!isTz && this.ctx.stage.im_type === imType.tz.value) {
@@ -97,6 +150,10 @@ module.exports = app => {
             this.stageImData.main = stageIm.ImData;
             if (isTz) {
                 this.stageImData.bills = stageIm.ImBillsData;
+                await this._setReportMemoryCache(stageImTz, tid, sid, this.ctx.stage.cacheTime, this.stageImData.main);
+                await this._setReportMemoryCache(stageImTzBills, tid, sid, this.ctx.stage.cacheTime, this.stageImData.bills);
+            } else {
+                await this._setReportMemoryCache(stageImZl, tid, sid, this.ctx.stage.cacheTime, this.stageImData.main);
             }
         }
 
@@ -125,6 +182,15 @@ module.exports = app => {
         }
 
         async getStageImTzData(tid, sid) {
+            await this._checkTender(tid);
+            await this._checkStage(sid);
+            const cache = await this._getReportMemoryCache('mem_stage_im_tz', tid, sid, this.ctx.stage.cacheTime);
+            if (cache) {
+                //console.log('cache');
+                return cache;
+            }
+
+            //console.log('build');
             if (!this.stageImData) {
                 this.stageImData = {};
                 try {
@@ -141,6 +207,11 @@ module.exports = app => {
         }
 
         async getStageImTzBillsData(tid, sid) {
+            await this._checkTender(tid);
+            await this._checkStage(sid);
+            const cache = await this._getReportMemoryCache('mem_stage_im_tz_bills', tid, sid, this.ctx.stage.cacheTime);
+            if (cache) return cache;
+
             if (!this.stageImData) {
                 this.stageImData = {};
                 try {
@@ -157,6 +228,11 @@ module.exports = app => {
         }
 
         async getStageImZlData(tid, sid) {
+            await this._checkTender(tid);
+            await this._checkStage(sid);
+            const cache = await this._getReportMemoryCache('mem_stage_im_zl', tid, sid, this.ctx.stage.cacheTime);
+            if (cache) return cache;
+
             this.stageImData = {};
             try {
                 await this._generateStageIm(tid, sid, false);

app/service/stage.js

@@ -297,6 +297,11 @@ module.exports = app => {
             return result.affectedRows === 1;
         }
 
+        async updateCacheTime(sid) {
+            const result = await this.db.update(this.tableName, {id: sid, cache_time_l: new Date()});
+            return result.affectedRows === 1;
+        }
+
         /**
          * 删除计量期
          *

app/service/stage_audit.js

@@ -202,6 +202,7 @@ module.exports = app => {
                     contract_tp: tpData.contract_tp,
                     qc_tp: tpData.qc_tp,
                     yf_tp: yfPay.tp,
+                    cache_time_r: this.ctx.stage.cache_time_l,
                 });
 
                 // 添加短信通知-需要审批提醒功能
@@ -239,6 +240,7 @@ module.exports = app => {
 
             const transaction = await this.db.beginTransaction();
             try {
+                console.log(checkData.opinion);
                 await transaction.update(this.tableName, {id: audit.id, status: checkData.checkType, opinion: checkData.opinion, end_time: time});
                 // 计算并合同支付最终数据
                 const yfPay = await this.ctx.service.stagePay.calcAllStagePays(this.ctx.stage, transaction);
@@ -254,6 +256,7 @@ module.exports = app => {
                         contract_tp: tpData.contract_tp,
                         qc_tp: tpData.qc_tp,
                         yf_tp: yfPay.tp,
+                        cache_time_r: this.ctx.stage.cache_time_l,
                     });
 
                     // 添加短信通知-需要审批提醒功能
@@ -282,6 +285,7 @@ module.exports = app => {
                         contract_tp: tpData.contract_tp,
                         qc_tp: tpData.qc_tp,
                         yf_tp: yfPay.tp,
+                        cache_time_r: this.ctx.stage.cache_time_l,
                     });
 
                     // 添加短信通知-审批通过提醒功能
@@ -347,6 +351,7 @@ module.exports = app => {
                     contract_tp: tpData.contract_tp,
                     qc_tp: tpData.qc_tp,
                     times: times + 1,
+                    cache_time_r: this.ctx.stage.cache_time_l,
                 });
                 // 拷贝新一次审核流程列表
                 await transaction.insert(this.tableName, auditors);
@@ -435,6 +440,7 @@ module.exports = app => {
                 // 同步 期信息
                 await transaction.update(this.ctx.service.stage.tableName, {
                     id: stageId, status: checkData.checkType,
+                    cache_time_r: this.ctx.stage.cache_time_l,
                 });
 
                 // 添加短信通知-需要审批提醒功能
@@ -644,6 +650,7 @@ module.exports = app => {
                 // 同步 期信息
                 await transaction.update(this.ctx.service.stage.tableName, {
                     id: stageId, status: auditConst.status.checking,
+                    cache_time_r: this.ctx.stage.cache_time_l,
                 });
 
                 // 添加短信通知-需要审批提醒功能

config/config.default.js

@@ -71,7 +71,7 @@ module.exports = appInfo => {
     };
 
     // 缓存时间
-    config.cacheTime = 1800;
+    config.cacheTime = 3600 * 24 * 31; // 31天 计量一期的时间，估计为1月
 
     // 安全性配置
     config.security = {

test/app/service/report_memory.test.js

@@ -21,6 +21,11 @@ describe('test/app/service/report_memory.test.js', () => {
             project: 'T201711273363',
             project_password: 'mai654321',
         };
+        // const postData = {
+        //     account: 'chente',
+        //     project: 'T201711273363',
+        //     project_password: '123456',
+        // };
         ctx.session = {};
         const loginResult = yield ctx.service.projectAccount.accountLogin(postData, 2);
         assert(loginResult);
@@ -35,9 +40,11 @@ describe('test/app/service/report_memory.test.js', () => {
         const mainData = yield ctx.service.reportMemory.getStageImTzData(12, stage.id);
         const billsData = yield ctx.service.reportMemory.getStageImTzBillsData(12, stage.id);
         if (mainData instanceof Array) {
+            console.log('mem_stage_im_tz.length:' + mainData.length);
             yield ctx.helper.saveBufferFile(JSON.stringify(mainData,"","\t"), ctx.app.baseDir + '/mem_stage_im_tz.json');
         }
         if (billsData instanceof Array) {
+            console.log('mem_stage_im_tz_bills.length:' + billsData.length);
             yield ctx.helper.saveBufferFile(JSON.stringify(billsData,"","\t"), ctx.app.baseDir + '/mem_stage_im_tz_bills.json');
         }
     });