|
@@ -137,18 +137,18 @@ const TIME_OUT = 60000;
|
|
|
const axiosInstance = axios.create({
|
|
|
baseURL: 'http://www.cqsgczjxx.org/',
|
|
|
timeout: TIME_OUT,
|
|
|
-/* proxy: {
|
|
|
- host: "127.0.0.1", port: "8888" // Fiddler抓包,需要打开Fiddler否则会报connect error
|
|
|
- }, */
|
|
|
+ /* proxy: {
|
|
|
+ host: "127.0.0.1", port: "8888" // Fiddler抓包,需要打开Fiddler否则会报connect error
|
|
|
+ }, */
|
|
|
headers: {
|
|
|
- 'Cache-Control': 'max-age=0',
|
|
|
- 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
|
|
|
- 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36',
|
|
|
- 'Accept': 'application/json, text/javascript, */*; q=0.01',
|
|
|
- 'X-Requested-With': 'XMLHttpRequest',
|
|
|
- 'Accept-Encoding': 'gzip, deflate',
|
|
|
- 'Accept-Language': 'zh-TW,zh;q=0.9,en-US;q=0.8,en;q=0.7,zh-CN;q=0.6',
|
|
|
- // 'Cookie': 'ASP.NET_SessionId=uozdrp0hep5x344vq153muju'
|
|
|
+ 'Cache-Control': 'max-age=0',
|
|
|
+ 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
|
|
|
+ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.141 Safari/537.36',
|
|
|
+ 'Accept': 'application/json, text/javascript, */*; q=0.01',
|
|
|
+ 'X-Requested-With': 'XMLHttpRequest',
|
|
|
+ 'Accept-Encoding': 'gzip, deflate',
|
|
|
+ 'Accept-Language': 'zh-TW,zh;q=0.9,en-US;q=0.8,en;q=0.7,zh-CN;q=0.6',
|
|
|
+ // 'Cookie': 'ASP.NET_SessionId=uozdrp0hep5x344vq153muju'
|
|
|
},
|
|
|
// responseType: 'json'
|
|
|
});
|
|
@@ -159,9 +159,9 @@ axiosInstance.interceptors.response.use(function (response) {
|
|
|
}, function (error) {
|
|
|
// 对响应错误做点什么
|
|
|
if (error.message.includes('timeout')) {
|
|
|
- return Promise.reject(`目标网络超时,请稍后再试。(${TIME_OUT}ms)`);
|
|
|
+ return Promise.reject(`目标网络超时,请稍后再试。(${TIME_OUT}ms)`);
|
|
|
} else {
|
|
|
- return Promise.reject(error);
|
|
|
+ return Promise.reject(error);
|
|
|
}
|
|
|
});
|
|
|
|
|
@@ -191,12 +191,12 @@ function month2quarter(period) {
|
|
|
|
|
|
function setTimeoutSync(handle, time) {
|
|
|
return new Promise((resolve, reject) => {
|
|
|
- setTimeout(() => {
|
|
|
- if (handle && typeof handle === 'function') {
|
|
|
- handle();
|
|
|
- }
|
|
|
- resolve();
|
|
|
- }, time);
|
|
|
+ setTimeout(() => {
|
|
|
+ if (handle && typeof handle === 'function') {
|
|
|
+ handle();
|
|
|
+ }
|
|
|
+ resolve();
|
|
|
+ }, time);
|
|
|
});
|
|
|
}
|
|
|
|
|
@@ -245,7 +245,7 @@ async function queryPrice(period, area, groupType, classify) {
|
|
|
option: 0,
|
|
|
token: ''
|
|
|
};
|
|
|
- const res = await post('/QueryInfoPrice', body);
|
|
|
+ const res = await post('/QueryInfoPrice', body);
|
|
|
return res && res.data && res.data.Data && res.data.Data._Items || [];
|
|
|
}
|
|
|
|
|
@@ -256,7 +256,7 @@ async function queryArea(period, groupType) {
|
|
|
period: period.replace('-', ''),
|
|
|
token: ''
|
|
|
};
|
|
|
- const res = await post('/QueryArea', body);
|
|
|
+ const res = await post('/QueryArea', body);
|
|
|
const areaData = res && res.data && res.data.Data && res.data.Data._Items || [];
|
|
|
return areaData.map(item => item.Area);
|
|
|
}
|
|
@@ -313,7 +313,7 @@ async function crawlBetonMaterial(period) {
|
|
|
const rst = [];
|
|
|
for (const area of areas) {
|
|
|
const priceItems = await queryPrice(period, area, groupType);
|
|
|
- const item = { area, data: [ { classify: '预拌商品砂浆', priceItems }] };
|
|
|
+ const item = { area, data: [{ classify: '预拌商品砂浆', priceItems }] };
|
|
|
rst.push(item);
|
|
|
}
|
|
|
return rst;
|
|
@@ -351,49 +351,49 @@ async function crawlGardenMateiral(period) {
|
|
|
const unit = 'CM';
|
|
|
const duplicateReg = /-/;
|
|
|
Object
|
|
|
- .entries(groupedData)
|
|
|
- .forEach(([kind, items]) => {
|
|
|
- const classItem = { classify: kind, priceItems: [], subClass: [] };
|
|
|
- rootClass.subClass.push(classItem);
|
|
|
- items.forEach(item => {
|
|
|
- // 拼接规格型号
|
|
|
- const specsList = [];
|
|
|
- if (item.Height) {
|
|
|
+ .entries(groupedData)
|
|
|
+ .forEach(([kind, items]) => {
|
|
|
+ const classItem = { classify: kind, priceItems: [], subClass: [] };
|
|
|
+ rootClass.subClass.push(classItem);
|
|
|
+ items.forEach(item => {
|
|
|
+ // 拼接规格型号
|
|
|
+ const specsList = [];
|
|
|
+ if (item.Height) {
|
|
|
specsList.push(`高度${item.Height}${unit}`);
|
|
|
- }
|
|
|
- if (item.TrunkDiameter) {
|
|
|
+ }
|
|
|
+ if (item.TrunkDiameter) {
|
|
|
specsList.push(`干径${item.TrunkDiameter}${unit}`);
|
|
|
- }
|
|
|
- if (item.TopDiameter) {
|
|
|
+ }
|
|
|
+ if (item.TopDiameter) {
|
|
|
specsList.push(`冠径${item.TopDiameter}${unit}`);
|
|
|
- }
|
|
|
- if (item.BranchHeight) {
|
|
|
+ }
|
|
|
+ if (item.BranchHeight) {
|
|
|
specsList.push(`分枝高${item.BranchHeight}${unit}`);
|
|
|
- }
|
|
|
- item.Model = specsList.join(' ');
|
|
|
- const isDuplicate = duplicateReg.test(item.TaxPrice) || duplicateReg.test(item.NoTaxPrice);
|
|
|
- if (isDuplicate) {
|
|
|
- // 分成最高低价最高价数据
|
|
|
- const taxPriceList = item.TaxPrice ? item.TaxPrice.split('-') : [''];
|
|
|
- const noTaxPriceList = item.NoTaxPrice ? item.NoTaxPrice.split('-') : [''];
|
|
|
- const minItem = {
|
|
|
- ...item,
|
|
|
- Name: `${item.Name}-最低价`,
|
|
|
- TaxPrice: taxPriceList[0],
|
|
|
- NoTaxPrice: noTaxPriceList[0]
|
|
|
- };
|
|
|
- const maxItem = {
|
|
|
- ...item,
|
|
|
- Name: `${item.Name}-最高价`,
|
|
|
- TaxPrice: taxPriceList[1] || '',
|
|
|
- NoTaxPrice: noTaxPriceList[1] || ''
|
|
|
- };
|
|
|
- classItem.priceItems.push(minItem, maxItem);
|
|
|
- } else {
|
|
|
- classItem.priceItems.push(item);
|
|
|
- }
|
|
|
+ }
|
|
|
+ item.Model = specsList.join(' ');
|
|
|
+ const isDuplicate = duplicateReg.test(item.TaxPrice) || duplicateReg.test(item.NoTaxPrice);
|
|
|
+ if (isDuplicate) {
|
|
|
+ // 分成最高低价最高价数据
|
|
|
+ const taxPriceList = item.TaxPrice ? item.TaxPrice.split('-') : [''];
|
|
|
+ const noTaxPriceList = item.NoTaxPrice ? item.NoTaxPrice.split('-') : [''];
|
|
|
+ const minItem = {
|
|
|
+ ...item,
|
|
|
+ Name: `${item.Name}-最低价`,
|
|
|
+ TaxPrice: taxPriceList[0],
|
|
|
+ NoTaxPrice: noTaxPriceList[0]
|
|
|
+ };
|
|
|
+ const maxItem = {
|
|
|
+ ...item,
|
|
|
+ Name: `${item.Name}-最高价`,
|
|
|
+ TaxPrice: taxPriceList[1] || '',
|
|
|
+ NoTaxPrice: noTaxPriceList[1] || ''
|
|
|
+ };
|
|
|
+ classItem.priceItems.push(minItem, maxItem);
|
|
|
+ } else {
|
|
|
+ classItem.priceItems.push(item);
|
|
|
+ }
|
|
|
+ });
|
|
|
});
|
|
|
- });
|
|
|
return rst;
|
|
|
}
|
|
|
|
|
@@ -470,7 +470,7 @@ async function crawlGeneralMaterial(period) {
|
|
|
*/
|
|
|
function getPeriodData(from, to) {
|
|
|
if (from > to) {
|
|
|
- return null;
|
|
|
+ return null;
|
|
|
}
|
|
|
// 根据区间获取期数列表
|
|
|
const reg = /(\d+)-(\d+)/;
|
|
@@ -496,15 +496,15 @@ function getPeriodData(from, to) {
|
|
|
'10': '10月',
|
|
|
'11': '11月',
|
|
|
'12': '12月',
|
|
|
-};
|
|
|
+ };
|
|
|
while (curYear <= toYear && curMonth <= toMonth) {
|
|
|
- list.push(`${curYear}年-${monthMap[curMonth]}`);
|
|
|
- if (curMonth === 12) {
|
|
|
- curYear++;
|
|
|
- curMonth = 1;
|
|
|
- } else {
|
|
|
- curMonth++;
|
|
|
- }
|
|
|
+ list.push(`${curYear}年-${monthMap[curMonth]}`);
|
|
|
+ if (curMonth === 12) {
|
|
|
+ curYear++;
|
|
|
+ curMonth = 1;
|
|
|
+ } else {
|
|
|
+ curMonth++;
|
|
|
+ }
|
|
|
}
|
|
|
return list;
|
|
|
}
|
|
@@ -514,16 +514,16 @@ async function areaPatch(compilationID) {
|
|
|
const areaData = await priceInfoAreaModel.find({ compilationID, serialNo: null }).lean();
|
|
|
const bulks = [];
|
|
|
areaData.forEach(areaItem => {
|
|
|
- const serialNo = defaultAreas.indexOf(areaItem.name) + 1;
|
|
|
- bulks.push({
|
|
|
- updateOne: {
|
|
|
- filter: { ID: areaItem.ID },
|
|
|
- update: { serialNo }
|
|
|
- }
|
|
|
- });
|
|
|
+ const serialNo = defaultAreas.indexOf(areaItem.name) + 1;
|
|
|
+ bulks.push({
|
|
|
+ updateOne: {
|
|
|
+ filter: { ID: areaItem.ID },
|
|
|
+ update: { serialNo }
|
|
|
+ }
|
|
|
+ });
|
|
|
});
|
|
|
if (bulks.length) {
|
|
|
- await priceInfoAreaModel.bulkWrite(bulks);
|
|
|
+ await priceInfoAreaModel.bulkWrite(bulks);
|
|
|
}
|
|
|
}
|
|
|
|
|
@@ -550,7 +550,8 @@ async function save(allData, period, compilationID) {
|
|
|
// 将各部分数据按照地区进行合并
|
|
|
const areaMap = {};
|
|
|
allData.forEach(({ area, data }) => {
|
|
|
- (areaMap[area] || (areaMap[area] = [])).push(...data);
|
|
|
+ const areaName = `重庆市-${area}`;
|
|
|
+ (areaMap[areaName] || (areaMap[areaName] = [])).push(...data);
|
|
|
});
|
|
|
const libData = { period, compilationID, ID: v1(), name: `信息价(${period})`, createDate: Date.now() };
|
|
|
const curAreas = await priceInfoAreaModel.find({ compilationID }).sort({ serialNo: 1 }).lean();
|
|
@@ -590,7 +591,7 @@ async function save(allData, period, compilationID) {
|
|
|
});
|
|
|
}
|
|
|
});
|
|
|
-
|
|
|
+
|
|
|
}
|
|
|
if (areaData.length) {
|
|
|
await priceInfoAreaModel.insertMany(areaData);
|
|
@@ -614,35 +615,35 @@ async function save(allData, period, compilationID) {
|
|
|
async function crawlData(from, to, compilationID) {
|
|
|
let curPeriod;
|
|
|
try {
|
|
|
- const periods = getPeriodData(from, to);
|
|
|
- if (!periods || !periods.length) {
|
|
|
- throw '无效的期数区间。';
|
|
|
- }
|
|
|
- // 地区补丁
|
|
|
- await areaPatch(compilationID);
|
|
|
- // 一期一期爬取数据
|
|
|
- for (const period of periods) {
|
|
|
- const labourData = await crawlLabour(period);
|
|
|
- const localeData = await crawlLocaleMaterial(period);
|
|
|
- const betonData = await crawlBetonMaterial(period);
|
|
|
- const generalData = await crawlGeneralMaterial(period);
|
|
|
- const allData = [...labourData, ...localeData, ...betonData, ...generalData];
|
|
|
- if (!allData.length) {
|
|
|
- throw `${period}无有效数据`;
|
|
|
- }
|
|
|
- await save(allData, period, compilationID);
|
|
|
- curPeriod = period;
|
|
|
+ const periods = getPeriodData(from, to);
|
|
|
+ if (!periods || !periods.length) {
|
|
|
+ throw '无效的期数区间。';
|
|
|
+ }
|
|
|
+ // 地区补丁
|
|
|
+ await areaPatch(compilationID);
|
|
|
+ // 一期一期爬取数据
|
|
|
+ for (const period of periods) {
|
|
|
+ const labourData = await crawlLabour(period);
|
|
|
+ const localeData = await crawlLocaleMaterial(period);
|
|
|
+ const betonData = await crawlBetonMaterial(period);
|
|
|
+ const generalData = await crawlGeneralMaterial(period);
|
|
|
+ const allData = [...labourData, ...localeData, ...betonData, ...generalData];
|
|
|
+ if (!allData.length) {
|
|
|
+ throw `${period}无有效数据`;
|
|
|
}
|
|
|
+ await save(allData, period, compilationID);
|
|
|
+ curPeriod = period;
|
|
|
+ }
|
|
|
} catch (err) {
|
|
|
- console.log(err);
|
|
|
- // 错误时提示已经成功爬取的期数
|
|
|
- let errTip = '';
|
|
|
- if (curPeriod) {
|
|
|
- errTip += `\n成功爬取期数为:${periods[0]}到${curPeriod}`;
|
|
|
- }
|
|
|
- const errStr = String(err) + errTip;
|
|
|
- console.log(`err`);
|
|
|
- console.log(errStr);
|
|
|
- throw errStr;
|
|
|
+ console.log(err);
|
|
|
+ // 错误时提示已经成功爬取的期数
|
|
|
+ let errTip = '';
|
|
|
+ if (curPeriod) {
|
|
|
+ errTip += `\n成功爬取期数为:${periods[0]}到${curPeriod}`;
|
|
|
+ }
|
|
|
+ const errStr = String(err) + errTip;
|
|
|
+ console.log(`err`);
|
|
|
+ console.log(errStr);
|
|
|
+ throw errStr;
|
|
|
}
|
|
|
}
|