const mongoose = require('mongoose'); const uuidV1 = require('uuid/v1'); const { CRAWL_LOG_KEY, ProcessStatus } = require('../../../public/constants/price_info_constant'); const priceInfoLibModel = mongoose.model('std_price_info_lib'); const priceInfoClassModel = mongoose.model('std_price_info_class'); const priceInfoItemModel = mongoose.model('std_price_info_items'); const priceInfoAreaModel = mongoose.model('std_price_info_areas'); const compilationModel = mongoose.model('compilation'); const importLogsModel = mongoose.model('import_logs'); async function getLibs(query) { return await priceInfoLibModel.find(query).lean(); } async function createLib(name, period, compilationID) { // 将2020-01变成2020年01月 const reg = /(\d{4})-(\d{2})/; const formattedPeriod = period.replace(reg, '$1年-$2月'); const lib = { ID: uuidV1(), name, period: formattedPeriod, compilationID, createDate: Date.now(), }; await priceInfoLibModel.create(lib); return lib; } async function updateLib(query, updateData) { await priceInfoLibModel.update(query, updateData); } async function deleteLib(libID) { await priceInfoClassModel.remove({ libID }); await priceInfoItemModel.remove({ libID }); await priceInfoLibModel.remove({ ID: libID }); } async function processChecking(key) { const logData = key ? await importLogsModel.findOne({ key }) : await importLogsModel.findOne({ key: CRAWL_LOG_KEY }); if (!logData) { return { status: ProcessStatus.FINISH }; } if (logData.status === ProcessStatus.FINISH || logData.status === ProcessStatus.ERROR) { await importLogsModel.remove({ key: logData.key }); } return { status: logData.status, errorMsg: logData.errorMsg || '', key: logData.key }; } // 爬取数据 async function crawlDataByCompilation(compilationID, from, to) { if (!compilationID) { throw '无有效费用定额。'; } const compilationData = await compilationModel.findOne({ _id: mongoose.Types.ObjectId(compilationID) }, 'overWriteUrl').lean(); if (!compilationData || !compilationData.overWriteUrl) { throw '无有效费用定额。'; } // 从overWriteUrl提取并组装爬虫文件 const reg = /\/([^/]+)\.js/; const matched = compilationData.overWriteUrl.match(reg); const crawlURL = `${matched[1]}_price_crawler.js`; let crawlData; try { const crawler = require(`../../../web/over_write/crawler/${crawlURL}`); crawlData = crawler.crawlData; } catch (e) { throw '该费用定额无可用爬虫方法。' } //await crawlData(from, to); // 异步不等结果,结果由checking来获取 crawlDataByMiddleware(crawlData, from, to, compilationID); } // 爬取数据中间件,主要处理checking初始化 async function crawlDataByMiddleware(crawlFunc, from, to, compilationID) { const logUpdateData = { status: ProcessStatus.FINISH }; try { const logData = { key: CRAWL_LOG_KEY, content: '正在爬取数据,请稍候……', status: ProcessStatus.START, create_time: Date.now() }; await importLogsModel.create(logData); await crawlFunc(from, to, compilationID); } catch (err) { console.log(err); logUpdateData.errorMsg = String(err); logUpdateData.status = ProcessStatus.ERROR; } finally { await importLogsModel.update({ key: CRAWL_LOG_KEY }, logUpdateData); } } // 导入excel数据,格式如下 // 格式1: //地区 分类 编码 名称 规格型号 单位 不含税价 含税价 //江北区 黑色及有色金属 热轧光圆钢筋 φ6(6.5) 3566.37 4030 //江北区 木、竹材料及其制品 柏木门套线 60×10 8.76 9.9 // 格式2: //地区 分类 编码 名称 规格型号 不含税价 含税价 //江北区 黑色及有色金属 热轧光圆钢筋 φ6(6.5) 3566.37 4030 // 柏木门套线 60×10 8.76 9.9 // 沥青混凝土 AC-13 982.3 1110 // //北碚区 木、竹材料及其制品 热轧光圆钢筋 φ6(6.5) 3566.37 4030 async function importExcelData(libID, sheetData) { const libs = await getLibs({ ID: libID }); const compilationID = libs[0].compilationID; // 建立区映射表:名称-ID映射、ID-名称映射 const areaList = await getAreas(compilationID); const areaMap = {}; areaList.forEach(({ ID, name }) => { areaMap[name] = ID; areaMap[ID] = name; }); // 建立分类映射表:地区名称@分类名称:ID映射 /* const classMap = {}; const classList = await getClassData(libID); classList.forEach(({ ID, areaID, name }) => { const areaName = areaMap[areaID] || ''; classMap[`${areaName}@${name}`] = ID; }); */ // 第一行获取行映射 const colMap = {}; for (let col = 0; col < sheetData[0].length; col++) { const cellText = sheetData[0][col]; switch (cellText) { case '地区': colMap.area = col; break; case '分类': colMap.class = col; break; case '编码': colMap.code = col; break; case '名称': colMap.name = col; break; case '规格型号': colMap.specs = col; break; case '单位': colMap.unit = col; break; case '不含税价': colMap.noTaxPrice = col; break; case '含税价': colMap.taxPrice = col; break; } } // 提取数据 const data = []; const classData = []; const areaClassDataMap = {}; let curAreaName; let curClassName; let curClassID; for (let row = 1; row < sheetData.length; row++) { const areaName = sheetData[row][colMap.area] || ''; const className = sheetData[row][colMap.class] || ''; const code = sheetData[row][colMap.code] || ''; const name = sheetData[row][colMap.name] || ''; const specs = sheetData[row][colMap.specs] || ''; const unit = sheetData[row][colMap.unit] || ''; const noTaxPrice = sheetData[row][colMap.noTaxPrice] || ''; const taxPrice = sheetData[row][colMap.taxPrice] || ''; if (!className && !code && !name && !specs && !noTaxPrice && !taxPrice) { // 认为是空数据 continue; } if (areaName && areaName !== curAreaName) { curAreaName = areaName; } const areaID = areaMap[curAreaName]; if (!areaID) { continue; } if (className && className !== curClassName) { curClassName = className; const classItem = { libID, areaID, ID: uuidV1(), ParentID: '-1', NextSiblingID: '-1', name: curClassName }; curClassID = classItem.ID; classData.push(classItem); (areaClassDataMap[areaID] || (areaClassDataMap[areaID] = [])).push(classItem); const preClassItem = areaClassDataMap[areaID][areaClassDataMap[areaID].length - 2]; if (preClassItem) { preClassItem.NextSiblingID = classItem.ID; } } if (!curClassID) { continue; } data.push({ ID: uuidV1(), compilationID, libID, areaID, classID: curClassID, period: libs[0].period, code, name, specs, unit, noTaxPrice, taxPrice }); } if (classData.length) { await priceInfoClassModel.remove({ libID }); await priceInfoClassModel.insertMany(classData); } if (data.length) { await priceInfoItemModel.remove({ libID }); await priceInfoItemModel.insertMany(data); } else { throw 'excel没有有效数据。' } } /* async function importExcelData(libID, sheetData) { const libs = await getLibs({ ID: libID }); const compilationID = libs[0].compilationID; // 建立区映射表:名称-ID映射、ID-名称映射 const areaList = await getAreas(compilationID); const areaMap = {}; areaList.forEach(({ ID, name }) => { areaMap[name] = ID; areaMap[ID] = name; }); // 建立分类映射表:地区名称@分类名称:ID映射 const classMap = {}; const classList = await getClassData(libID); classList.forEach(({ ID, areaID, name }) => { const areaName = areaMap[areaID] || ''; classMap[`${areaName}@${name}`] = ID; }); // 第一行获取行映射 const colMap = {}; for (let col = 0; col < sheetData[0].length; col++) { const cellText = sheetData[0][col]; switch (cellText) { case '地区': colMap.area = col; break; case '分类': colMap.class = col; break; case '编码': colMap.code = col; break; case '名称': colMap.name = col; break; case '规格型号': colMap.specs = col; break; case '单位': colMap.unit = col; break; case '不含税价': colMap.noTaxPrice = col; break; case '含税价': colMap.taxPrice = col; break; } } // 提取数据 const data = []; let curAreaName; let curClassName; for (let row = 1; row < sheetData.length; row++) { const areaName = sheetData[row][colMap.area] || ''; const className = sheetData[row][colMap.class] || ''; const code = sheetData[row][colMap.code] || ''; const name = sheetData[row][colMap.name] || ''; const specs = sheetData[row][colMap.specs] || ''; const unit = sheetData[row][colMap.unit] || ''; const noTaxPrice = sheetData[row][colMap.noTaxPrice] || ''; const taxPrice = sheetData[row][colMap.taxPrice] || ''; if (!code && !name && !specs && !noTaxPrice && !taxPrice) { // 认为是空数据 continue; } if (areaName && areaName !== curAreaName) { curAreaName = areaName; } if (className && className !== curClassName) { curClassName = className; } const areaID = areaMap[curAreaName]; if (!areaID) { continue; } const classID = classMap[`${curAreaName}@${curClassName}`]; if (!classID) { continue; } data.push({ ID: uuidV1(), compilationID, libID, areaID, classID, period: libs[0].period, code, name, specs, unit, noTaxPrice, taxPrice }); } if (data.length) { await priceInfoItemModel.remove({ libID }); await priceInfoItemModel.insertMany(data); } else { throw 'excel没有有效数据。' } } */ // 获取费用定额的地区数据 async function getAreas(compilationID) { return await priceInfoAreaModel.find({ compilationID }, '-_id ID name serialNo').lean(); } async function updateAres(updateData) { const bulks = []; updateData.forEach(({ ID, name }) => bulks.push({ updateOne: { filter: { ID }, update: { name } } })); if (bulks.length) { await priceInfoAreaModel.bulkWrite(bulks); } } async function insertAreas(insertData) { await priceInfoAreaModel.insertMany(insertData); } async function deleteAreas(deleteData) { await priceInfoClassModel.remove({ areaID: { $in: deleteData } }); await priceInfoItemModel.remove({ areaID: { $in: deleteData } }); await priceInfoAreaModel.remove({ ID: { $in: deleteData } }); } async function getClassData(libID, areaID) { if (libID && areaID) { return await priceInfoClassModel.find({ libID, areaID }, '-_id').lean(); } if (libID) { return await priceInfoClassModel.find({ libID }, '-_id').lean(); } if (areaID) { return await priceInfoClassModel.find({ areaID }, '-_id').lean(); } } async function getPriceData(classIDList) { return await priceInfoItemModel.find({ classID: { $in: classIDList } }, '-_id').lean(); } const UpdateType = { UPDATE: 'update', DELETE: 'delete', CREATE: 'create', }; async function editPriceData(postData) { const bulks = []; postData.forEach(data => { if (data.type === UpdateType.UPDATE) { bulks.push({ updateOne: { filter: { ID: data.ID }, update: { ...data.data } } }); } else if (data.type === UpdateType.DELETE) { bulks.push({ deleteOne: { filter: { ID: data.ID } } }); } else { bulks.push({ insertOne: { document: data.data } }); } }); if (bulks.length) { await priceInfoItemModel.bulkWrite(bulks); } } async function editClassData(updateData) { const bulks = []; const deleteIDList = []; updateData.forEach(({ type, filter, update, document }) => { if (type === UpdateType.UPDATE) { bulks.push({ updateOne: { filter, update } }); } else if (type === UpdateType.DELETE) { deleteIDList.push(filter.ID); bulks.push({ deleteOne: { filter } }); } else { bulks.push({ insertOne: { document } }); } }); if (deleteIDList.length) { await priceInfoItemModel.remove({ classID: { $in: deleteIDList } }); } if (bulks.length) { await priceInfoClassModel.bulkWrite(bulks); } } module.exports = { getLibs, createLib, updateLib, deleteLib, processChecking, crawlDataByCompilation, importExcelData, getAreas, updateAres, insertAreas, deleteAreas, getClassData, getPriceData, editPriceData, editClassData }