perf: 优化导入excel清单速度

modules/all_models/stdBills_itemCharacter.js

@@ -8,7 +8,7 @@ const stdBills_itemCharacter = new Schema({
         code: Number,
         content: String,
         itemValue: Array,
-        billsLibId: Number,
+        billsLibId: {type:Number, index: true},
         deleted: Boolean
     },
     {versionKey: false}

modules/all_models/stdBills_jobContent.js

@@ -8,7 +8,7 @@ const stdBills_jobContent = new Schema({
         id: Number,
         code: Number,
         content: String,
-        billsLibId: Number,
+        billsLibId: {type:Number, index: true},
         deleted: Boolean
     },
     {versionKey: false}

modules/main/controllers/bills_controller.js

@@ -227,21 +227,38 @@ module.exports = {
                 if(compressData === null){
                     throw 'excel没有对应数据'
                 }
+                const date1 = Date.now();
                 let importData = JSON.parse(LZString.decompressFromUTF16(compressData));
                 //匹配的清单库
+                const date2 = Date.now();
+                console.log(date2 - date1, 'decompressFromUTF16');
                 const billsLibId = fields.billsLibId !== undefined && fields.billsLibId.length > 0 && fields.billsLibId[0]? parseInt(fields.billsLibId[0]) : null;
                 let stdBills = [], stdJobs = [], stdCharacters = [];
+                const stdBillMap = {};
+                const stdJobMap = {};
+                const stdCharacterMap = {};
                 if(billsLibId){
-                    stdBills = await stdBillsModel.find({billsLibId: billsLibId, deleted: false}, '-_id code jobs items engineering billsLibId ruleText');
-                    stdJobs = await stdBillJobsModel.find({billsLibId: billsLibId, deleted: false});
-                    stdCharacters = await stdBillCharacterModel.find({billsLibId: billsLibId, deleted: false});
+                    stdBills = await stdBillsModel.find({billsLibId: billsLibId}, '-_id code jobs items engineering billsLibId ruleText');
+                    stdJobs = await stdBillJobsModel.find({billsLibId: billsLibId});
+                    stdCharacters = await stdBillCharacterModel.find({billsLibId: billsLibId});
+                    // 标准数据映射
+                    stdBills.forEach(bill => {
+                        stdBillMap[bill.code] = bill;
+                    });
+                    stdJobs.forEach(job => {
+                        stdJobMap[job.id] = job;
+                    });
+                    stdCharacters.forEach(character => {
+                        stdCharacterMap[character.id] = character;
+                    });
                 }
-                let stdData = {stdBills: stdBills, stdJobs: stdJobs, stdCharacters: stdCharacters};
+                let stdData = {stdBills: stdBills, stdJobs: stdJobs, stdCharacters: stdCharacters, stdBillMap, stdJobMap, stdCharacterMap};
+                console.log(Date.now() - date2, 'getStd');
                 //导入表
                 let importDateA = +new Date();
                 for(let position in importData){
-                    if(importData[position].length > 0){
-                        let updateFrontData = await importSheet(position, importData[position], req.session.sessionUser.id, projectID, stdData);
+                    if(importData[position] && importData[position].insert && importData[position].insert.length > 0){
+                        let updateFrontData = await importSheet(position, importData[position].insert, importData[position].remove, projectID, stdData);
                         if(updateFrontData){
                             responseData.data.push(updateFrontData);
                         }
@@ -271,7 +288,7 @@ module.exports = {
     }
 };
 
-async function importSheet(position, excelBills, userID, projectID, stdData){
+async function importSheet(position, excelBills, removeData, projectID, stdData){
         //导入位置的有固定行
         let flag = getImportFlag(position);
         if(!flag){
@@ -307,9 +324,13 @@ async function importSheet(position, excelBills, userID, projectID, stdData){
         //将excel清单数据转换成完整清单数据（设置ParentID、匹配标准清单库）
         parseToCompleteBills(excelBills, fixedBill, stdData);
         //删除相关数据
-        let deleteDatas = await billsData.deepDeleteBill([fixedBill], userID);
+        const date = Date.now();
+        let deleteDatas = await billsData.deepDeleteBill(removeData.rBillIDs, removeData.rRationIDs);
+        const date1 = Date.now();
+        console.log(date1 - date, 'deepDeleteBill');
         //新增清单数据
         await billsData.importBills(excelBills);
+        console.log(Date.now() - date1, 'importBills');
         // 如果导入招标、控制价文件、每个分项底下自动生成一条空定额
         const rations = [];
         excelBills.forEach(bills => {
@@ -355,45 +376,42 @@ function parseToCompleteBills(excelBills, fixedBills, stdData){
             bills.ParentID = rootID;
         }
         delete bills.nodeType;
+        const date = Date.now();
         matchStdBill(bills, stdData);
+        console.log(Date.now() - date, 'matchStdBill');
     }
 
     //excel数据与标准库数据匹配，根据清单前九位编码匹配，匹配成功则获取标准清单对应的工程专业、特征及内容
     function matchStdBill(excelBill, stdData){
         let isMatch = false;
-        let regExp = /^\d{12}$/g;
         if(excelBill.code.length >8){
             let nineCode = excelBill.code.substr(0, 9);
-            for(let stdBill of stdData.stdBills){
-                //set programID
-                if(nineCode == stdBill.code){
-                    isMatch = true;
-                    excelBill.programID = stdBill.engineering ? stdBill.engineering : null;
-                    excelBill.billsLibId = stdBill.billsLibId ? stdBill.billsLibId : null;
-                    excelBill.ruleText = stdBill.ruleText ? stdBill.ruleText : '';
-                    //set jobContent and itemCharacter
-                    let tempJob = [], tempCharacter = [];
-                    for(let billJob of stdBill.jobs){
-                        for(let stdJob of stdData.stdJobs) {
-                            if (billJob.id == stdJob.id) {
-                                tempJob.push({isChecked: false, serialNo: billJob.serialNo, content: stdJob.content});
-                            }
-                        }
+            const stdBill = stdData.stdBillMap[nineCode];
+            if (stdBill) {
+                isMatch = true;
+                excelBill.programID = stdBill.engineering ? stdBill.engineering : null;
+                excelBill.billsLibId = stdBill.billsLibId ? stdBill.billsLibId : null;
+                excelBill.ruleText = stdBill.ruleText ? stdBill.ruleText : '';
+                //set jobContent and itemCharacter
+                let tempJob = [], tempCharacter = [];
+                for(let billJob of stdBill.jobs){
+                    const stdJob = stdData.stdJobMap[billJob.id];
+                    if (stdJob) {
+                        tempJob.push({isChecked: false, serialNo: billJob.serialNo, content: stdJob.content});
                     }
-                    for(let billCharacter of stdBill.items){
-                        for(let stdCharacter of stdData.stdCharacters){
-                            if(billCharacter.id == stdCharacter.id){
-                                let eigenvalue = [];
-                                for(let eValue of stdCharacter.itemValue){
-                                    eigenvalue.push({isSelected: false, value: eValue.value});
-                                }
-                                tempCharacter.push({isChecked: false, serialNo: billCharacter.serialNo, character: stdCharacter.content, eigenvalue: eigenvalue});
-                            }
+                }
+                for(let billCharacter of stdBill.items){
+                    const stdCharacter = stdData.stdCharacterMap[billCharacter.id];
+                    if (stdCharacter) {
+                        let eigenvalue = [];
+                        for(let eValue of stdCharacter.itemValue){
+                            eigenvalue.push({isSelected: false, value: eValue.value});
                         }
+                        tempCharacter.push({isChecked: false, serialNo: billCharacter.serialNo, character: stdCharacter.content, eigenvalue: eigenvalue});
                     }
-                    excelBill.jobContent = tempJob;
-                    excelBill.itemCharacter = tempCharacter;
                 }
+                excelBill.jobContent = tempJob;
+                excelBill.itemCharacter = tempCharacter;
             }
         }
         if(!isMatch && excelBill.type === billType.FX){//分项不为空，同时在标准清单中不匹配，则识别为补项

modules/main/models/bills.js

@@ -166,40 +166,18 @@ class billsModel extends baseModel {
         return bills.update(findSet, update);
     };
     async importBills(bills){
-        let operations = [];
+        /* let operations = [];
         for(let bill of bills){
             operations.push({insertOne: {document: bill}});
+        } */
+        if (bills.length) {
+            await this.model.insertMany(bills)
         }
-        return await this.model.bulkWrite(operations);
     }
     //删除清单节点的所有子节点及其他附带数据
-    async deepDeleteBill(bills, userID){
-        let bill_ids = [],
-            ration_ids = [];
+    async deepDeleteBill(bill_ids, ration_ids){
         let me = this;
-        async function findBillsChildren(bills){
-            if(bills.length > 0){
-                let ids = getIDs(bills);
-                bill_ids = bill_ids.concat(ids);
-                let findBills = await me.model.find({ParentID: {$in: ids}, deleteInfo: null});
-                await findBillsChildren(findBills);
-            }
-        }
-        function getIDs(datas){
-            let ids = [];
-            for(let data of datas){
-                ids.push(data.ID);
-            }
-            return ids;
-        }
-        await findBillsChildren(bills);
-        //剔除第一个节点
-        bill_ids = bill_ids.slice(1);
-        //获取删除清单下的所有定额
-        let rations = await rationModel.find({billsItemID: {$in: bill_ids}, deleteInfo: null}, 'ID');
-        ration_ids = ration_ids.concat(getIDs(rations));
         //deep delete datas
-        let deleteInfo = {deleted: true, deleteDateTime: new Date(), deleteBy: userID};
         if(bill_ids.length > 0){
             //删除bills
             await me.model.deleteMany({ID: {$in: bill_ids}});

web/building_saas/main/js/views/importBills.js

@@ -608,7 +608,37 @@ const importBills = (function(){
         return false;
     }
 
+    // 获取需要删除的D
+    function getRemoveIDs(position, positionBills) {
+        const rBillIDs = [];
+        const rRationIDs = [];
+        const rst = { rBillIDs, rRationIDs };
+        if (!positionBills || !positionBills.length) {
+            return rst;
+        }
+        let flag;
+        if (position === 'fbfx') {
+            flag = fixedFlag.SUB_ENGINERRING;
+        } else if (position === 'jscsxm') {
+            flag = fixedFlag.CONSTRUCTION_TECH;
+        } else {
+            flag = fixedFlag.CONSTRUCTION_ORGANIZATION;
+        }
+        const fixedNode = calcTools.getNodeByFlag(flag);
+        if (!fixedNode) {
+            return rst;
+        }
+        fixedNode.getPosterity().forEach(node => {
+            if (node.sourceType === 'bills') {
+                rBillIDs.push(node.data.ID);
+            } else {      
+                rRationIDs.push(node.data.ID);
+            }
+        });
+        return rst;
+    }
+
 
 
-    return {setImportSheetsInfo, getImportSheetsInfo, getImportSheets, getImportData, excelHasValidBills}
+    return {setImportSheetsInfo, getImportSheetsInfo, getImportSheets, getImportData, excelHasValidBills, getRemoveIDs}
 })();

web/building_saas/main/js/views/project_view.js

@@ -3184,15 +3184,20 @@ $('#importConfirm').click(function () {
             throw '请勾选要导入的表';
         }
         let importSheets = importBills.getImportSheets(importJson.sheets, importSheetsInfo, fileType);
-        console.log(`importSheets`);
-        console.log(importSheets);
         let invalidSheetsInfo = importSheets.invalidSheets.join('、');
         //获取同类表的有效数据
         let importBillsData = importBills.getImportData(importSheets.validSheets, projectID);
         if(!importBills.excelHasValidBills(importBillsData)){
             throw 'excel无有效数据'
         }
-        let compressData = LZString.compressToUTF16(JSON.stringify(importBillsData));
+        const postData = {};
+        Object.keys(importBillsData).forEach(position => {
+            postData[position] = {
+                insert: importBillsData[position],
+                remove: importBills.getRemoveIDs(position, importBillsData[position]),
+            }
+        })
+        let compressData = LZString.compressToUTF16(JSON.stringify(postData));
         formData.append('compressData', compressData);
         let eDate = +new Date();
         console.log(`解析excel数据时间：${eDate - sDate}`);