|
@@ -20,6 +20,7 @@ const compilationModel = mongoose.model('compilation');
|
|
const priceInfoLibModel = mongoose.model('std_price_info_lib');
|
|
const priceInfoLibModel = mongoose.model('std_price_info_lib');
|
|
const priceInfoClassModel = mongoose.model('std_price_info_class');
|
|
const priceInfoClassModel = mongoose.model('std_price_info_class');
|
|
const priceInfoItemModel = mongoose.model('std_price_info_items');
|
|
const priceInfoItemModel = mongoose.model('std_price_info_items');
|
|
|
|
+const priceInfoAreaModel = mongoose.model('std_price_info_areas');
|
|
|
|
|
|
const isDebug = true;
|
|
const isDebug = true;
|
|
|
|
|
|
@@ -580,13 +581,13 @@ async function crawlMixedData(period) {
|
|
* @param {String} libID - 库ID
|
|
* @param {String} libID - 库ID
|
|
* @param {String} classID - 所属分类ID
|
|
* @param {String} classID - 所属分类ID
|
|
* @param {String} period - 期数 eg:2020年01月
|
|
* @param {String} period - 期数 eg:2020年01月
|
|
- * @param {String} area - 地区
|
|
|
|
|
|
+ * @param {String} areaID - 地区ID
|
|
* @param {String} compilationID - 费用定额ID
|
|
* @param {String} compilationID - 费用定额ID
|
|
* @param {Array<object>} items - 爬取的信息价源数据
|
|
* @param {Array<object>} items - 爬取的信息价源数据
|
|
* @param {Number} tableType - 表格类型
|
|
* @param {Number} tableType - 表格类型
|
|
* @return {Array<obejct>}
|
|
* @return {Array<obejct>}
|
|
*/
|
|
*/
|
|
-function transformPriceItems(libID, classID, period, area, compilationID, items, tableType) {
|
|
|
|
|
|
+function transformPriceItems(libID, classID, period, areaID, compilationID, items, tableType) {
|
|
const rst = [];
|
|
const rst = [];
|
|
if (tableType === TableType.GARDEN) {
|
|
if (tableType === TableType.GARDEN) {
|
|
// 有的数据 高度(CM) | 干径(CM) | 冠径(CM) | 分枝高(CM) | 不含税价(元) = ‘’ | 14-17 | 大于400 | 200-300 | 430-780
|
|
// 有的数据 高度(CM) | 干径(CM) | 冠径(CM) | 分枝高(CM) | 不含税价(元) = ‘’ | 14-17 | 大于400 | 200-300 | 430-780
|
|
@@ -630,10 +631,10 @@ function transformPriceItems(libID, classID, period, area, compilationID, items,
|
|
taxPrice: taxPriceList[1] || '',
|
|
taxPrice: taxPriceList[1] || '',
|
|
noTaxPrice: noTaxPriceList[1] || ''
|
|
noTaxPrice: noTaxPriceList[1] || ''
|
|
};
|
|
};
|
|
- rst.push(transfromPriceItem(libID, classID, period, area, compilationID, minItem));
|
|
|
|
- rst.push(transfromPriceItem(libID, classID, period, area, compilationID, maxItem));
|
|
|
|
|
|
+ rst.push(transfromPriceItem(libID, classID, period, areaID, compilationID, minItem));
|
|
|
|
+ rst.push(transfromPriceItem(libID, classID, period, areaID, compilationID, maxItem));
|
|
} else {
|
|
} else {
|
|
- rst.push(transfromPriceItem(libID, classID, period, area, compilationID, item));
|
|
|
|
|
|
+ rst.push(transfromPriceItem(libID, classID, period, areaID, compilationID, item));
|
|
}
|
|
}
|
|
})
|
|
})
|
|
} else {
|
|
} else {
|
|
@@ -662,13 +663,13 @@ function transformPriceItems(libID, classID, period, area, compilationID, items,
|
|
taxPrice: taxPriceList[index] || taxPriceList[0],
|
|
taxPrice: taxPriceList[index] || taxPriceList[0],
|
|
noTaxPrice: noTaxPriceList[index] || noTaxPriceList[0]
|
|
noTaxPrice: noTaxPriceList[index] || noTaxPriceList[0]
|
|
};
|
|
};
|
|
- if (area) {
|
|
|
|
- newItem.area = area;
|
|
|
|
|
|
+ if (areaID) {
|
|
|
|
+ newItem.areaID = areaID;
|
|
}
|
|
}
|
|
- rst.push(transfromPriceItem(libID, classID, period, area, compilationID, newItem));
|
|
|
|
|
|
+ rst.push(transfromPriceItem(libID, classID, period, areaID, compilationID, newItem));
|
|
});
|
|
});
|
|
} else {
|
|
} else {
|
|
- rst.push(transfromPriceItem(libID, classID, period, area, compilationID, item));
|
|
|
|
|
|
+ rst.push(transfromPriceItem(libID, classID, period, areaID, compilationID, item));
|
|
}
|
|
}
|
|
});
|
|
});
|
|
}
|
|
}
|
|
@@ -676,10 +677,10 @@ function transformPriceItems(libID, classID, period, area, compilationID, items,
|
|
}
|
|
}
|
|
|
|
|
|
// 转换单条的价格数据
|
|
// 转换单条的价格数据
|
|
-function transfromPriceItem(libID, classID, period, area, compilationID, item) {
|
|
|
|
|
|
+function transfromPriceItem(libID, classID, period, areaID, compilationID, item) {
|
|
// 源数据中的规格型号存在多个无意义的空格,合并为一个
|
|
// 源数据中的规格型号存在多个无意义的空格,合并为一个
|
|
const reg = /\s{2,}/g;
|
|
const reg = /\s{2,}/g;
|
|
- item.specs = item.specs.replace(reg, ' ');
|
|
|
|
|
|
+ item.specs = item.specs ? item.specs.replace(reg, ' ') : '';
|
|
return {
|
|
return {
|
|
ID: uuidV1(),
|
|
ID: uuidV1(),
|
|
libID,
|
|
libID,
|
|
@@ -693,7 +694,7 @@ function transfromPriceItem(libID, classID, period, area, compilationID, item) {
|
|
remark: item.remark || '',
|
|
remark: item.remark || '',
|
|
// 以下冗余数据为方便前台信息价功能处理
|
|
// 以下冗余数据为方便前台信息价功能处理
|
|
period,
|
|
period,
|
|
- area,
|
|
|
|
|
|
+ areaID,
|
|
compilationID,
|
|
compilationID,
|
|
}
|
|
}
|
|
}
|
|
}
|
|
@@ -703,13 +704,17 @@ function transfromPriceItem(libID, classID, period, area, compilationID, item) {
|
|
* @param {String} period - 日期: 2020年01月
|
|
* @param {String} period - 日期: 2020年01月
|
|
* @param {String} compilationID - 费用定额ID
|
|
* @param {String} compilationID - 费用定额ID
|
|
* @param {Object} generalData - 主要材料{ building, garden, energy }
|
|
* @param {Object} generalData - 主要材料{ building, garden, energy }
|
|
- * @return {Object} { libData, classData, priceData }
|
|
|
|
|
|
+ * @return {Object} { libData, classData, priceData, compilationAreas }
|
|
*/
|
|
*/
|
|
-function transfromGeneralData(period, compilationID, generalData) {
|
|
|
|
|
|
+async function transfromGeneralData(period, compilationID, generalData) {
|
|
const area = '通用';
|
|
const area = '通用';
|
|
|
|
+ // 爬取数据的时候,地区数据先匹配名称,如果费用定额已有此地区,不新增
|
|
|
|
+ const matchedArea = await priceInfoAreaModel.findOne({ compilationID, name: area }).lean();
|
|
|
|
+ const areaID = matchedArea && matchedArea.ID || uuidV1();
|
|
|
|
+ const compilationAreas = [];
|
|
const libData = {
|
|
const libData = {
|
|
ID: uuidV1(),
|
|
ID: uuidV1(),
|
|
- name: `${area}信息价(${period})`,
|
|
|
|
|
|
+ name: `信息价(${period})`,
|
|
period,
|
|
period,
|
|
areas: [],
|
|
areas: [],
|
|
compilationID,
|
|
compilationID,
|
|
@@ -735,11 +740,11 @@ function transfromGeneralData(period, compilationID, generalData) {
|
|
// 绿色节能分类数据:绿色、节能建筑工程材料
|
|
// 绿色节能分类数据:绿色、节能建筑工程材料
|
|
const energyData = [{ materialClass: '绿色、节能建筑工程材料', items: energy }];
|
|
const energyData = [{ materialClass: '绿色、节能建筑工程材料', items: energy }];
|
|
handleClassAndItems(energyData, TableType.ENERGY);
|
|
handleClassAndItems(energyData, TableType.ENERGY);
|
|
- // 有数据才将地区push入areas中,必须返回非空的libData,后续转换地区数据需要这个libData(一期只生成一个库)
|
|
|
|
- if (classData.length || priceData.length) {
|
|
|
|
- libData.areas.push(area);
|
|
|
|
|
|
+ // 有数据才将地区push入areas中(费用定额共用)
|
|
|
|
+ if ((classData.length || priceData.length) && !matchedArea) {
|
|
|
|
+ compilationAreas.push({ compilationID, ID: areaID, name: area })
|
|
}
|
|
}
|
|
- return { libData, classData, priceData };
|
|
|
|
|
|
+ return { libData, classData, priceData, compilationAreas };
|
|
|
|
|
|
function handleClassAndItems(sourceData, tableType) {
|
|
function handleClassAndItems(sourceData, tableType) {
|
|
if (!sourceData) {
|
|
if (!sourceData) {
|
|
@@ -752,7 +757,7 @@ function transfromGeneralData(period, compilationID, generalData) {
|
|
NextSiblingID: treeData && treeData.NextSiblingID || '-1',
|
|
NextSiblingID: treeData && treeData.NextSiblingID || '-1',
|
|
name: materialClass,
|
|
name: materialClass,
|
|
libID: libData.ID,
|
|
libID: libData.ID,
|
|
- area,
|
|
|
|
|
|
+ areaID,
|
|
};
|
|
};
|
|
// 设置上一个节点数据的NextID
|
|
// 设置上一个节点数据的NextID
|
|
let count = 1;
|
|
let count = 1;
|
|
@@ -768,7 +773,7 @@ function transfromGeneralData(period, compilationID, generalData) {
|
|
classData.push(classItem);
|
|
classData.push(classItem);
|
|
// 转换价格数据
|
|
// 转换价格数据
|
|
if (items && items.length) {
|
|
if (items && items.length) {
|
|
- const newItems = transformPriceItems(libData.ID, classItem.ID, period, area, compilationID, items, tableType);
|
|
|
|
|
|
+ const newItems = transformPriceItems(libData.ID, classItem.ID, period, areaID, compilationID, items, tableType);
|
|
newItems.forEach(item => priceData.push(item));
|
|
newItems.forEach(item => priceData.push(item));
|
|
}
|
|
}
|
|
});
|
|
});
|
|
@@ -784,8 +789,9 @@ function transfromGeneralData(period, compilationID, generalData) {
|
|
* @param {Object} libData - 当前期数库数据
|
|
* @param {Object} libData - 当前期数库数据
|
|
* @param {Array<object>} areaData - 各区县地方材料工地价格
|
|
* @param {Array<object>} areaData - 各区县地方材料工地价格
|
|
* @param {Array<object>} mixedData - 预拌砂浆信息价格
|
|
* @param {Array<object>} mixedData - 预拌砂浆信息价格
|
|
|
|
+ * @return {Object}
|
|
*/
|
|
*/
|
|
-function transformAreaData(period, compilationID, libData, areaData, mixedData) {
|
|
|
|
|
|
+async function transformAreaData(period, compilationID, libData, areaData, mixedData) {
|
|
// 根据地区进行分类
|
|
// 根据地区进行分类
|
|
const data = [];
|
|
const data = [];
|
|
const hashMap = {}; // 保证地区顺序跟网页爬取数据的顺序一致。(object for in无法保证顺序)
|
|
const hashMap = {}; // 保证地区顺序跟网页爬取数据的顺序一致。(object for in无法保证顺序)
|
|
@@ -820,10 +826,15 @@ function transformAreaData(period, compilationID, libData, areaData, mixedData)
|
|
}
|
|
}
|
|
buildData(areaData);
|
|
buildData(areaData);
|
|
buildData(mixedData);
|
|
buildData(mixedData);
|
|
|
|
+ const compilationAreas = [];
|
|
const classData = [];
|
|
const classData = [];
|
|
const priceData = [];
|
|
const priceData = [];
|
|
- data.forEach(({ area, subData }) => {
|
|
|
|
- libData.areas.push(area);
|
|
|
|
|
|
+ for (const { area, subData } of data) {
|
|
|
|
+ const matchedArea = await priceInfoAreaModel.findOne({ compilationID, name: area }).lean();
|
|
|
|
+ const areaID = matchedArea && matchedArea.ID || uuidV1();
|
|
|
|
+ if (!matchedArea) {
|
|
|
|
+ compilationAreas.push({ compilationID, ID: areaID, name: area });
|
|
|
|
+ }
|
|
let preClass;
|
|
let preClass;
|
|
subData.forEach(subItem => {
|
|
subData.forEach(subItem => {
|
|
if (!subItem) {
|
|
if (!subItem) {
|
|
@@ -836,18 +847,18 @@ function transformAreaData(period, compilationID, libData, areaData, mixedData)
|
|
NextSiblingID: '-1',
|
|
NextSiblingID: '-1',
|
|
name: className,
|
|
name: className,
|
|
libID: libData.ID,
|
|
libID: libData.ID,
|
|
- area,
|
|
|
|
|
|
+ areaID,
|
|
};
|
|
};
|
|
classData.push(classItem);
|
|
classData.push(classItem);
|
|
if (preClass) {
|
|
if (preClass) {
|
|
preClass.NextSiblingID = classItem.ID;
|
|
preClass.NextSiblingID = classItem.ID;
|
|
}
|
|
}
|
|
preClass = classItem;
|
|
preClass = classItem;
|
|
- const newItems = transformPriceItems(libData.ID, classItem.ID, period, area, compilationID, items, TableType.AREA);
|
|
|
|
|
|
+ const newItems = transformPriceItems(libData.ID, classItem.ID, period, areaID, compilationID, items, TableType.AREA);
|
|
newItems.forEach(item => priceData.push(item));
|
|
newItems.forEach(item => priceData.push(item));
|
|
});
|
|
});
|
|
- });
|
|
|
|
- return { classData, priceData };
|
|
|
|
|
|
+ }
|
|
|
|
+ return { classData, priceData, compilationAreas };
|
|
}
|
|
}
|
|
|
|
|
|
/**
|
|
/**
|
|
@@ -866,12 +877,13 @@ async function save(period, generalData, areaData, mixedData) {
|
|
}
|
|
}
|
|
const compilationID = compilation._id;
|
|
const compilationID = compilation._id;
|
|
// 转换数据
|
|
// 转换数据
|
|
- const generalSaveData = transfromGeneralData(period, compilationID, generalData);
|
|
|
|
|
|
+ const generalSaveData = await transfromGeneralData(period, compilationID, generalData);
|
|
const libData = generalSaveData.libData;
|
|
const libData = generalSaveData.libData;
|
|
- const areaSaveData = transformAreaData(period, compilationID, libData, areaData, mixedData);
|
|
|
|
|
|
+ const areaSaveData = await transformAreaData(period, compilationID, libData, areaData, mixedData);
|
|
// 入库
|
|
// 入库
|
|
const classData = [...generalSaveData.classData, ...areaSaveData.classData];
|
|
const classData = [...generalSaveData.classData, ...areaSaveData.classData];
|
|
const priceData = [...generalSaveData.priceData, ...areaSaveData.priceData];
|
|
const priceData = [...generalSaveData.priceData, ...areaSaveData.priceData];
|
|
|
|
+ const compilationAreas = [...generalSaveData.compilationAreas, ...areaSaveData.compilationAreas]
|
|
// 删除已有的相同期数数据
|
|
// 删除已有的相同期数数据
|
|
const originalLibs = await priceInfoLibModel.find({ period }, '-_id ID').lean();
|
|
const originalLibs = await priceInfoLibModel.find({ period }, '-_id ID').lean();
|
|
const originalLibIDList = originalLibs.reduce((acc, cur) => {
|
|
const originalLibIDList = originalLibs.reduce((acc, cur) => {
|
|
@@ -893,6 +905,9 @@ async function save(period, generalData, areaData, mixedData) {
|
|
if (libData) {
|
|
if (libData) {
|
|
await priceInfoLibModel.insertMany([libData]);
|
|
await priceInfoLibModel.insertMany([libData]);
|
|
}
|
|
}
|
|
|
|
+ if (compilationAreas) {
|
|
|
|
+ await priceInfoAreaModel.insertMany(compilationAreas);
|
|
|
|
+ }
|
|
}
|
|
}
|
|
|
|
|
|
/**
|
|
/**
|