refactor:爬取信息价由长请求更改为checking检测机制

modules/all_models/import_logs.js

@@ -0,0 +1,26 @@
+/**
+ * Created by zhang on 2019/12/27.
+ */
+const mongoose = require('mongoose');
+
+let Schema = mongoose.Schema;
+let collectionName = 'import_logs';
+
+let modelSchema = {
+    // 日志类型
+    key: {type: String, index: true},
+    // 日志简单内容
+    content: String,
+    // 关联用户id
+    userID: String,
+    // 费用定额Id
+    compilationID: String,
+    // 状态
+    status:String,
+    // 建设项目ID
+    projectID: Array,
+    // 创建时间
+    create_time: Number,
+    errorMsg:""
+};
+mongoose.model(collectionName, new Schema(modelSchema, {versionKey: false, collection: collectionName}));

modules/price_info_lib/controllers/index.js

@@ -93,13 +93,23 @@ class PriceInfoController extends BaseController {
         }
     }
 
+    async processChecking(req, res) {
+        try {
+            const { key } = JSON.parse(req.body.data);
+            const data = await facade.processChecking(key);
+            res.json({ error: 0, message: 'processChecking', data });
+        } catch (err) {
+            res.json({ error: 1, message: err.toString() });
+        }
+    }
+
     // 爬取数据
     async crawlData(req, res) {
         try {
             const { from, to, compilationID } = JSON.parse(req.body.data);
-            res.setTimeout(1000 * 60 * 60 * 2); // 不设置的话，处理时间过长，会触发默认的响应超时，报错(前端报错，后台还继续在处理)
+            //res.setTimeout(1000 * 60 * 60 * 2); // 不设置的话，处理时间过长，会触发默认的响应超时，报错(前端报错，后台还继续在处理)
             await facade.crawlDataByCompilation(compilationID, from, to);
-            res.json({ error: 0, message: 'crawl success' });
+            res.json({ error: 0, message: 'crawl processing' });
         } catch (err) {
             console.log(err);
             res.json({ error: 1, message: err.toString() });

modules/price_info_lib/facade/index.js

@@ -1,11 +1,13 @@
 const mongoose = require('mongoose');
 const uuidV1 = require('uuid/v1');
+const { CRAWL_LOG_KEY, ProcessStatus } = require('../../../public/constants/price_info_constant');
 
 const priceInfoLibModel = mongoose.model('std_price_info_lib');
 const priceInfoClassModel = mongoose.model('std_price_info_class');
 const priceInfoItemModel = mongoose.model('std_price_info_items');
 const priceInfoAreaModel = mongoose.model('std_price_info_areas');
 const compilationModel = mongoose.model('compilation');
+const importLogsModel = mongoose.model('import_logs');
 
 async function getLibs(query) {
     return await priceInfoLibModel.find(query).lean();
@@ -36,6 +38,19 @@ async function deleteLib(libID) {
     await priceInfoLibModel.remove({ ID: libID });
 }
 
+async function processChecking(key) {
+    const logData = key
+        ? await importLogsModel.findOne({ key })
+        : await importLogsModel.findOne({ key: CRAWL_LOG_KEY });
+    if (!logData) {
+        return { status: ProcessStatus.FINISH };
+    }
+    if (logData.status === ProcessStatus.FINISH || logData.status === ProcessStatus.ERROR) {
+        await importLogsModel.remove({ key: logData.key });
+    }
+    return { status: logData.status, errorMsg: logData.errorMsg || '', key: logData.key };
+}
+
 // 爬取数据
 async function crawlDataByCompilation(compilationID, from, to) {
     if (!compilationID) {
@@ -56,14 +71,36 @@ async function crawlDataByCompilation(compilationID, from, to) {
     } catch (e) {
         throw '该费用定额无可用爬虫方法。'
     }
-    await crawlData(from, to);
+    //await crawlData(from, to);
+    // 异步不等结果，结果由checking来获取
+    crawlDataByMiddleware(crawlData, from, to);
+}
+
+// 爬取数据中间件，主要处理checking初始化
+async function crawlDataByMiddleware(crawlFunc, from, to) {
+    const logUpdateData = { status: ProcessStatus.FINISH };
+    try {
+        const logData = {
+            key: CRAWL_LOG_KEY,
+            content: '正在爬取数据，请稍候……',
+            status: ProcessStatus.START,
+            create_time: Date.now()
+        };
+        await importLogsModel.create(logData);
+        await crawlFunc(from, to);
+    } catch (err) {
+        logUpdateData.errorMsg = err;
+        logUpdateData.status = ProcessStatus.ERROR;
+    } finally {
+        await importLogsModel.update({ key: CRAWL_LOG_KEY }, logUpdateData);
+    }
 }
 
 // 导入excel数据，格式如下
 // 格式1：
-//地区	    分类	        编码	名称	        规格型号	不含税价	含税价
-//江北区	黑色及有色金属		    热轧光圆钢筋	  φ6（6.5）	 3566.37	4030
-//江北区	木、竹材料及其制品		柏木门套线	      60×10	     8.76	    9.9
+//地区	    分类	        编码	名称	        规格型号    单位	不含税价	含税价
+//江北区	黑色及有色金属		    热轧光圆钢筋	  φ6（6.5）	         3566.37	4030
+//江北区	木、竹材料及其制品		柏木门套线	      60×10	             8.76	    9.9
 // 格式2：
 //地区	    分类	            编码	名称	        规格型号	    不含税价	含税价
 //江北区	黑色及有色金属		        热轧光圆钢筋	  φ6（6.5）	     3566.37	4030
@@ -72,7 +109,6 @@ async function crawlDataByCompilation(compilationID, from, to) {
 //						
 //北碚区	木、竹材料及其制品		    热轧光圆钢筋	  φ6（6.5）	      3566.37	4030
 async function importExcelData(libID, sheetData) {
-    console.log(sheetData);
     const libs = await getLibs({ ID: libID });
     const compilationID = libs[0].compilationID;
     // 建立区映射表：名称-ID映射、ID-名称映射
@@ -292,6 +328,7 @@ module.exports = {
     createLib,
     updateLib,
     deleteLib,
+    processChecking,
     crawlDataByCompilation,
     importExcelData,
     getAreas,

modules/price_info_lib/routes/index.js

@@ -13,6 +13,7 @@ module.exports = function (app) {
     router.post("/addLib", priceInfoController.auth, priceInfoController.init, priceInfoController.addLib);
     router.post("/renameLib", priceInfoController.auth, priceInfoController.init, priceInfoController.renameLib);
     router.post("/deleteLib", priceInfoController.auth, priceInfoController.init, priceInfoController.deleteLib);
+    router.post("/processChecking", priceInfoController.auth, priceInfoController.init, priceInfoController.processChecking);
     router.post("/crawlData", priceInfoController.auth, priceInfoController.init, priceInfoController.crawlData);
     router.post("/importExcel", priceInfoController.auth, priceInfoController.init, priceInfoController.importExcel);
 

public/constants/price_info_constant.js

@@ -0,0 +1,16 @@
+((factory) => {
+    if (typeof module !== 'undefined' && !module.nodeType) {
+        module.exports = factory();
+    } else {
+        window.PRICE_INFO_CONST = factory();
+    }
+})(() => {
+    return {
+        CRAWL_LOG_KEY: 'OPERATION_CRAWL_DATA',
+        ProcessStatus: {
+            START: 'start',
+            FINISH: 'finish',
+            ERROR: 'error'
+        }
+    };
+});

web/maintain/price_info_lib/html/main.html

@@ -224,4 +224,5 @@
 
 <script src="/public/web/lock_util.js"></script>
 <script src="/public/web/PerfectLoad.js"></script>
+<script src="/public/constants/price_info_constant.js"></script>
 <script src="/web/maintain/price_info_lib/js/main.js"></script>

web/maintain/price_info_lib/js/main.js

@@ -138,6 +138,56 @@ function handleImportConfirm() {
     }
 }
 
+const { ProcessStatus, CRAWL_LOG_KEY } = window.PRICE_INFO_CONST;
+
+const CHECKING_TIME = 5000;
+// 检测爬取、导入是否完成
+function processChecking(key, cb) {
+    checking();
+
+    function checking() {
+        ajaxPost('/priceInfo/processChecking', { key })
+            .then(handleResolve)
+            .catch(handleReject)
+    }
+
+    let timer;
+    function handleResolve({ key, status, errorMsg }) {
+        if (status === ProcessStatus.START) {
+            if (!$('#progressModal').is(':visible')) {
+                const title = key === CRAWL_LOG_KEY ? '爬取数据' : '导入数据';
+                const text = key === CRAWL_LOG_KEY ? '正在爬取数据，请稍候……' : '正在导入数据，请稍候……';
+                $.bootstrapLoading.progressStart(title, true);
+                $("#progress_modal_body").text(text);
+            }
+            timer = setTimeout(checking, CHECKING_TIME);
+        } else if (status === ProcessStatus.FINISH) {
+            if (timer) {
+                clearTimeout(timer);
+            }
+            $.bootstrapLoading.progressEnd();
+            if (cb) {
+                cb();
+            }
+        } else {
+            if (timer) {
+                clearTimeout(timer);
+            }
+            if (errorMsg) {
+                alert(errorMsg);
+            }
+            $.bootstrapLoading.progressEnd();
+        }
+    }
+    function handleReject(err) {
+        if (timer) {
+            clearInterval(timer);
+        }
+        alert(err);
+        $.bootstrapLoading.progressEnd();
+    }
+}
+
 const matched = window.location.search.match(/filter=(.+)/);
 const compilationID = matched && matched[1] || '';
 // 爬取数据确认
@@ -149,6 +199,19 @@ function handleCrawlConfirm() {
         return false;
     }
     $('#crawl').modal('hide');
+    ajaxPost('/priceInfo/crawlData', { from, to, compilationID }, 0) // 没有timeout
+        .then(() => {
+            processChecking(CRAWL_LOG_KEY, () => window.location.reload());
+        })
+}
+/* function handleCrawlConfirm() {
+    const from = $('#period-start').val();
+    const to = $('#period-end').val();
+    if (!periodReg.test(from) || !periodReg.test(to)) {
+        $('#crawl-error').show();
+        return false;
+    }
+    $('#crawl').modal('hide');
     $.bootstrapLoading.progressStart('爬取数据', true);
     $("#progress_modal_body").text('正在爬取数据，请稍候……');
     // 不用定时器的话，可能finally处理完后，进度条界面才显示，导致进度条界面没有被隐藏
@@ -162,10 +225,12 @@ function handleCrawlConfirm() {
                 .finally(() => $.bootstrapLoading.progressEnd());
         }
     }, 500);
-}
+} */
 
 const throttleTime = 1000;
 $(document).ready(function () {
+    processChecking();
+
     // 锁定、解锁
     $('.lock').click(function () {
         lockUtil.handleLockClick($(this));

web/over_write/js/chongqing_2018_price_crawler.js

@@ -15,6 +15,7 @@ const querystring = require('querystring');
 const uuidV1 = require('uuid/v1');
 const mongoose = require('mongoose');
 const { isDef } = require('../../../public/common_util');
+const { SSL_OP_SSLEAY_080_CLIENT_DH_BUG } = require('constants');
 
 const compilationModel = mongoose.model('compilation');
 const priceInfoLibModel = mongoose.model('std_price_info_lib');