Calculation/app/reports/util/rpt_calculation_data_util.js

'use strict';

/**
 * Created by Tony on 2019/6/18.
 * 计量报表数据提取class，是协助报表模板里指标字段自主提取数据的工具类
 */
const JV = require('../rpt_component/jpc_value_define');
const $JE = require('../rpt_component/jpc_rte');

const fs = require('fs');
const fsUtil = require('../../public/js/fsUtil');
const stringUtil = require('../../public/js/stringUtil');
const scMathUtil = require('../../public/js/scMathUtil');
const data_analyze_util = require('../../lib/rpt_data_analysis');
const _ = require('lodash');

const treeUtil = require('../public/treeUtil');

const RELATION_TABLES_KEYS = {
    project: { main_key: 'id' },
    tender: { main_key: 'id', foreign_key: { project: 'project_id' } },
    tender_info: { main_key: 'id', foreign_key: { project: 'pid', tender: 'tid' } },
    ledger: { main_key: 'id', isTree: true, tree_pid: 'ledger_pid', tree_id: 'ledger_id', foreign_key: { tender: 'tender_id' } },
};

class Rpt_Common {
    initialize(rpt_tpl, currentDataObj) {
        this.template = rpt_tpl;
        this.currentDataObj = currentDataObj;
    }
    Multiply(val1, val2, fixFormat) {
        const rst = [];
        let maxLen = val1.length;
        let minLen = val2.length;
        if (minLen > maxLen) {
            maxLen = maxLen + minLen; minLen = maxLen - minLen; maxLen = maxLen - minLen;
        }
        for (let i = 0; i < maxLen; i++) {
            let value = ((i < val1.length) ? val1[i] : val1[minLen - 1]) * ((i < val2.length) ? val2[i] : val2[minLen - 1]);
            if (value === null || value === undefined) {
                value = '0';
            }
            if (fixFormat) value = value.toFixed(fixFormat);
            rst.push(value);
        }
        return rst;
    }
    Divide(val1, val2, fixFormat) {
        const rst = [];
        let maxLen = val1.length;
        let minLen = val2.length;
        if (minLen > maxLen) {
            maxLen = maxLen + minLen; minLen = maxLen - minLen; maxLen = maxLen - minLen;
        }
        for (let i = 0; i < maxLen; i++) {
            let value = ((i < val1.length) ? val1[i] : val1[minLen - 1]) / ((i < val2.length) ? val2[i] : val2[minLen - 1]);
            if (fixFormat) value = value.toFixed(fixFormat);
            rst.push(value);
        }
        return rst;
    }
    Plus(val1, val2, fixFormat) {
        const rst = [];
        let maxLen = val1.length;
        let minLen = val2.length;
        if (minLen > maxLen) {
            maxLen = maxLen + minLen; minLen = maxLen - minLen; maxLen = maxLen - minLen;
        }
        for (let i = 0; i < maxLen; i++) {
            let value = ((i < val1.length) ? val1[i] : val1[minLen - 1]) + ((i < val2.length) ? val2[i] : val2[minLen - 1]);
            if (fixFormat) value = value.toFixed(fixFormat);
            rst.push(value);
        }
        return rst;
    }
    MultiPlus(arrVal, fixFormat) {
        const rst = [];
        for (let i = 0; i < arrVal.length; i++) {
            const valItem = arrVal[i];
            if (i === 0) {
                for (const dtl of valItem) {
                    let value = parseFloat(dtl);
                    if (fixFormat) value = value.toFixed(fixFormat);
                    rst.push(value);
                }
            } else {
                for (let j = 0; j < valItem.length; j++) {
                    if (j < rst.length) {
                        let value = rst[j] + valItem[j];
                        if (fixFormat) value = value.toFixed(fixFormat);
                        rst[j] = value;
                    } else {
                        let value = parseFloat(valItem[j]);
                        if (fixFormat) value = value.toFixed(fixFormat);
                        rst.push(value);
                    }
                }
            }
        }
        return rst;
    }
    Minus(val1, val2, fixFormat) {
        const rst = [];
        let maxLen = val1.length;
        let minLen = val2.length;
        if (minLen > maxLen) {
            maxLen = maxLen + minLen; minLen = maxLen - minLen; maxLen = maxLen - minLen;
        }
        for (let i = 0; i < maxLen; i++) {
            let value = ((i < val1.length) ? val1[i] : val1[minLen - 1]) - ((i < val2.length) ? val2[i] : val2[minLen - 1]);
            if (fixFormat) value = value.toFixed(fixFormat);
            rst.push(value);
        }
        return rst;
    }
    FormatString(arrVal, formatStr) {
        const rst = [];
        for (const val of arrVal) {
            rst.push(stringUtil.replaceAll(formatStr, '%S', val));
        }
        return rst;
    }
}

class Rpt_Data_Extractor {
    constructor() {
        this.COMMON = new Rpt_Common();
    }

    initialize(tpl) {
        this.rptTpl = tpl;
    }

    // 因为计量是采用传统的关系数据库(MySQL), 有一些表关系是需要先约定(如上下级关系，外键关系等)

    // 根据报表模板映射指标(非离散指标)的定义，罗列出所有需要用到的data对象key，作为数据请求的过滤依据
    getDataRequestFilter() {
        const rst = {};
        const memFieldKeys = {}; // 内存表优化用，需要获得相关内存表的指标请求，因为有些额外指标需要花资源去获得
        const tables = []; // 记录需要查询哪些表
        // const filters = []; // 记录过滤条件(预处理中的过滤)，目前先不处理
        const tpl = this.rptTpl;
        const pri_func_split_mem_fieldKey = function(field) {
            // 计量需要获取所有内存表指标的key，优化用
            if (memFieldKeys[field.TableName] === undefined) {
                memFieldKeys[field.TableName] = [];
            }
            const strs = field[JV.PROP_FIELD_EXP_MAP].split("', '");
            if (strs.length === 2) {
                const codeKey = strs[1].slice(0, strs[1].indexOf("'"));
                if (field.TableName.indexOf('mem_') === 0 && memFieldKeys[field.TableName].indexOf(codeKey) < 0) {
                    memFieldKeys[field.TableName].push(codeKey);
                }
            }
        };
        const pri_func_chk_filter = function(field) {
            // 计量的机制有所不同，数据分开保存在不同的表里，
            if (field.TableName && tables.indexOf(field.TableName) < 0) {
                tables.push(field.TableName);
            }
            pri_func_split_mem_fieldKey(field);
        };
        const pri_setup_filter = function(FIELD_LIST_KEY) {
            // console.log(tpl[JV.NODE_FIELD_MAP][FIELD_LIST_KEY]);
            if (tpl[JV.NODE_FIELD_MAP][FIELD_LIST_KEY]) {
                for (const field of tpl[JV.NODE_FIELD_MAP][FIELD_LIST_KEY]) {
                    pri_func_chk_filter(field);
                }
            }
        };
        pri_setup_filter(JV.NODE_DISCRETE_FIELDS);
        pri_setup_filter(JV.NODE_MASTER_FIELDS);
        pri_setup_filter(JV.NODE_DETAIL_FIELDS);
        pri_setup_filter(JV.NODE_MASTER_FIELDS_EX);
        pri_setup_filter(JV.NODE_DETAIL_FIELDS_EX);
        if (tpl[JV.NODE_MAP_DATA_HANDLE_INFO] && tpl[JV.NODE_MAP_DATA_HANDLE_INFO].length > 0) {
            // 计量的预处理会有所变化，没有那么多方式，这里只记录过滤
            for (const preHandle of tpl[JV.NODE_MAP_DATA_HANDLE_INFO]) {
                if (preHandle[JV.PROP_HANDLE_TYPE] === JV.PROP_HANDLE_TYPE_FILTER) {
                    if (tables.indexOf(preHandle[JV.PROP_DATA_KEY]) < 0) {
                        tables.push(preHandle[JV.PROP_DATA_KEY]);
                    }
                    // 目前还没想好有哪些过滤，骑驴找马吧
                }
            }
        }
        rst.tables = tables;
        rst.memFieldKeys = memFieldKeys;
        return rst;
    }

    // 装配数据(把收集到的数据，依据报表模板的指示，预处理(如：排序、过滤、合计)及装配到相关指标)
    assembleData(ctx, rawDataObj, baseDir) {
        const $PROJECT = { REPORT: {} };
        const tpl = this.rptTpl;
        this.COMMON.initialize(tpl, rawDataObj);
        $PROJECT.COMMON = this.COMMON;

        // console.log(rawDataObj);
        setupFunc($PROJECT.REPORT, rawDataObj, baseDir);

        if (tpl[JV.NODE_MAP_DATA_HANDLE_INFO]) {
            for (const preHandle of tpl[JV.NODE_MAP_DATA_HANDLE_INFO]) {
                const srcData = getModuleDataByKey(rawDataObj.prjData, preHandle[JV.PROP_DATA_KEY]);
                switch (preHandle[JV.PROP_HANDLE_TYPE]) {
                    case JV.PROP_HANDLE_TYPE_SORT:
                        // sortData(srcData, preHandle, rawDataObj.prjData);
                        break;
                    case JV.PROP_HANDLE_TYPE_FILTER:
                        // filterData(srcData, preHandle, rawDataObj.prjData);
                        break;
                    case JV.PROP_HANDLE_TYPE_PRE_DEFINED:
                        preDefineProcess(ctx, tpl, preHandle, rawDataObj);
                        break;
                    default:
                        break;
                }
            }
        }

        const rptDataObj = {};
        rptDataObj[JV.DATA_DISCRETE_DATA] = [];
        rptDataObj[JV.DATA_MASTER_DATA] = [];
        rptDataObj[JV.DATA_DETAIL_DATA] = [];
        rptDataObj[JV.DATA_MASTER_DATA_EX] = [];
        rptDataObj[JV.DATA_DETAIL_DATA_EX] = [];

        assembleFields(tpl[JV.NODE_FIELD_MAP][JV.NODE_DISCRETE_FIELDS], rptDataObj[JV.DATA_DISCRETE_DATA], $PROJECT);
        // console.log(JV.DATA_DISCRETE_DATA);
        // console.log(rptDataObj[JV.DATA_DISCRETE_DATA]);
        assembleFields(tpl[JV.NODE_FIELD_MAP][JV.NODE_MASTER_FIELDS], rptDataObj[JV.DATA_MASTER_DATA], $PROJECT);
        // console.log(JV.DATA_MASTER_DATA);
        // console.log(rptDataObj[JV.DATA_MASTER_DATA]);
        assembleFields(tpl[JV.NODE_FIELD_MAP][JV.NODE_DETAIL_FIELDS], rptDataObj[JV.DATA_DETAIL_DATA], $PROJECT);
        // console.log(JV.DATA_DETAIL_DATA);
        // console.log(rptDataObj[JV.DATA_DETAIL_DATA]);
        assembleFields(tpl[JV.NODE_FIELD_MAP][JV.NODE_MASTER_FIELDS_EX], rptDataObj[JV.DATA_MASTER_DATA_EX], $PROJECT);
        // console.log(JV.DATA_MASTER_DATA_EX);
        // console.log(rptDataObj[JV.DATA_MASTER_DATA_EX]);
        assembleFields(tpl[JV.NODE_FIELD_MAP][JV.NODE_DETAIL_FIELDS_EX], rptDataObj[JV.DATA_DETAIL_DATA_EX], $PROJECT);
        // console.log(JV.DATA_DETAIL_DATA_EX);
        // console.log(rptDataObj[JV.DATA_DETAIL_DATA_EX]);
        // fsUtil.writeObjToFile(rptDataObj, 'D:/GitHome/temp/insertedOriginalData.jsp');
        // fsUtil.writeObjToFile(rawDataObj, 'D:/GitHome/temp/insertedRawDataData.jsp');
        // fsUtil.writeObjToFile($PROJECT, 'D:/GitHome/temp/$PROJECTData.jsp');
        // fsUtil.writeObjToFile(tpl[JV.NODE_FIELD_MAP][JV.NODE_MASTER_FIELDS], 'D:/GitHome/temp/masterFieldsAfterAssemble.jsp');
        // fsUtil.writeObjToFile(tpl[JV.NODE_FIELD_MAP][JV.NODE_DETAIL_FIELDS], 'D:/GitHome/temp/detailFieldsAfterAssemble.jsp');
        // fsUtil.writeObjToFile(tpl[JV.NODE_FIELD_MAP][JV.NODE_DISCRETE_FIELDS], 'D:/GitHome/temp/discreteFieldsAfterAssemble.jsp');
        return rptDataObj;
    }

}

function getModuleDataByKey(prjData, key) {
    let rst = null;
    if (prjData) {
        for (const item of prjData) {
            if (item.moduleName === key) {
                rst = item;
                break;
            }
        }
    }
    return rst;
}

function filterData(sourceData, handleCfg, prjData) {
    let rstArr = [];
    const tempRstArr = [];
    for (const item of getActDataArr(sourceData)) {
        if (item._doc) {
            tempRstArr.push(item._doc);
        } else {
            tempRstArr.push(item);
        }
    }
    const private_chkVal = function(src, compVal, compStr) {
        let rst = true;
        switch (compStr) {
            case '==' :
                rst = (src == compVal);
                break;
            case '===' :
                rst = (src === compVal);
                break;
            case '>' :
                rst = (src > compVal);
                break;
            case '>=' :
                rst = (src >= compVal);
                break;
            case '<' :
                rst = (src < compVal);
                break;
            case '<=' :
                rst = (src <= compVal);
                break;
            case '!=' :
                rst = (src != compVal);
                break;
            case '!==' :
                rst = (src !== compVal);
                break;
            case 'in' :
                if (compVal instanceof Array) {
                    rst = compVal.indexOf(src) >= 0;
                } else {
                    // string，需要转类型
                    const newInCv = JSON.parse(compVal);
                    if (newInCv instanceof Array) {
                        rst = newInCv.indexOf(src) >= 0;
                    } else {
                        rst = false;
                    }
                }
                break;
            case 'not in':
                if (compVal instanceof Array) {
                    rst = compVal.indexOf(src) < 0;
                } else {
                    // string，需要转类型
                    const newNotInCv = JSON.parse(compVal);
                    if (newNotInCv instanceof Array) {
                        rst = (newNotInCv.indexOf(src) < 0);
                    } else {
                        rst = true;
                    }
                }
                break;
            default:
                rst = true;
        }
        return rst;
    };
    const private_chkArrVal = function(arr, key, compVal, compStr) {
        let rst = false;
        if (arr.length > 0) {
            for (const arrItem of arr) {
                if (arrItem[key] !== undefined) {
                    // 可以为null值去判断
                    rst = private_chkVal(arrItem[key], compVal, compStr);
                }
                if (rst) {
                    break;
                }
            }
        } else {
            // 在某些判断条件下(含有'非'判断)，如arr没有数组项，默认结果反而是true
            switch (compStr) {
                case '!=' :
                case '!==' :
                case 'not in':
                    rst = true;
                    break;
                default:
                    break;
            }
        }
        return rst;
    };
    const private_filter_compare = function(item, filterCfg) {
        const compareObj = {};
        let compRst = true;
        let curComparePrjData = null;
        let startIdx = 0;
        const private_ref_join = function(refKey, targetDataKey, targetPropertyKey) {
            let rst = null;
            let objDataArr = null;
            curComparePrjData = getModuleDataByKey(prjData, targetDataKey);
            try {
                if (curComparePrjData !== null) {
                    objDataArr = getActDataArr(curComparePrjData);
                    for (const dtl of objDataArr) {
                        if (item[refKey] === dtl[targetPropertyKey]) {
                            rst = dtl;
                            break;
                        }
                    }
                }
            } finally {
                curComparePrjData = null;
            }
            return rst;
        };
        for (const cfg of filterCfg[JV.PROP_FILTER_KEYS]) {
            if (cfg[JV.PROP_FILTER_COMPARE_VAL]) {
                // 比较key值
                const keys = cfg.key.split('.');
                if (keys.length > 1) {
                    let lastObj = item;
                    for (let i = 0; i < keys.length - 1; i++) {
                        if (keys[i].indexOf('ref_join(') === 0) {
                            const params = keys[i].slice(9, keys[i].length - 1).split(',');
                            if (params.length === 3) {
                                lastObj = private_ref_join(params[0], params[1], params[2]);
                            }
                            if (!(lastObj)) {
                                compRst = false;
                                break;
                            }
                        } else {
                            lastObj = item[keys[i]];
                            if (!(lastObj)) {
                                compRst = false;
                                break;
                            }
                        }
                    }
                    if (lastObj) {
                        if (lastObj instanceof Array) {
                            compRst = private_chkArrVal(lastObj, keys[keys.length - 1], cfg[JV.PROP_FILTER_COMPARE_VAL], cfg[JV.PROP_FILTER_CONDITION]);
                        } else {
                            compRst = private_chkVal(lastObj[keys[keys.length - 1]], cfg[JV.PROP_FILTER_COMPARE_VAL], cfg[JV.PROP_FILTER_CONDITION]);
                        }
                    }
                } else {
                    compRst = private_chkVal(item[cfg.key], cfg[JV.PROP_FILTER_COMPARE_VAL], cfg[JV.PROP_FILTER_CONDITION]);
                }
            } else if (cfg[JV.PROP_FILTER_COMPARE_OBJ] && cfg[JV.PROP_FILTER_COMPARE_OBJ_KEY]) {
                // 通过其他对象来过滤
                if (!curComparePrjData) {
                    curComparePrjData = getModuleDataByKey(prjData, cfg[JV.PROP_FILTER_COMPARE_OBJ]);
                }
                if (cfg[JV.PROP_FILTER_CONDITION] === 'in' || cfg[JV.PROP_FILTER_CONDITION] === 'not in') {
                    let compareArr = null;
                    if (!compareObj.hasOwnProperty(cfg[JV.PROP_FILTER_COMPARE_OBJ_KEY] + startIdx.toString())) {
                        compareObj[cfg[JV.PROP_FILTER_COMPARE_OBJ_KEY] + startIdx.toString()] = [];
                        compareArr = compareObj[cfg[JV.PROP_FILTER_COMPARE_OBJ_KEY] + startIdx.toString()];
                        for (const data of getActDataArr(curComparePrjData)) {
                            if (compareArr.indexOf(data[cfg[JV.PROP_FILTER_COMPARE_OBJ_KEY]]) < 0) {
                                compareArr.push(data[cfg[JV.PROP_FILTER_COMPARE_OBJ_KEY]]);
                            }
                        }
                    } else {
                        compareArr = compareObj[cfg[JV.PROP_FILTER_COMPARE_OBJ_KEY] + startIdx.toString()];
                    }
                    compRst = private_chkVal(item[cfg.key], compareArr, cfg[JV.PROP_FILTER_CONDITION]);
                } else {
                    for (const data of getActDataArr(curComparePrjData)) {
                        compRst = private_chkVal(item[cfg.key], data[cfg[JV.PROP_FILTER_COMPARE_OBJ_KEY]], cfg[JV.PROP_FILTER_CONDITION]);
                        if (compRst) break;
                    }
                }
            }
            startIdx++;
            if (!compRst) {
                break; // 有不符合条件的数据则退出(这里的判断条件是and关系)
            }
        }
        return compRst;
    };
    const private_sub_filter_compare = function(dtlItem, subFilters) {
        let cmpRst = false;
        for (const dtlCfg of subFilters) {
            cmpRst = private_filter_compare(dtlItem, dtlCfg);
            if (cmpRst) {
                if (dtlCfg[JV.PROP_OTHER_SUB_FILTER] && dtlCfg[JV.PROP_OTHER_SUB_FILTER].length > 0) {
                    cmpRst = private_sub_filter_compare(dtlItem, dtlCfg[JV.PROP_OTHER_SUB_FILTER]);
                    if (cmpRst) break;
                } else {
                    break;
                }
            }
        }
        return cmpRst;
    };

    for (const item of tempRstArr) {
        if (private_filter_compare(item, handleCfg)) {
            rstArr.push(item);
        }
    }
    if (handleCfg[JV.PROP_OTHER_SUB_FILTER] && handleCfg[JV.PROP_OTHER_SUB_FILTER].length > 0) {
        const newRstArr = [];
        for (const dtlItem of rstArr) {
            const cmpRst = private_sub_filter_compare(dtlItem, handleCfg[JV.PROP_OTHER_SUB_FILTER]);
            if (cmpRst) {
                newRstArr.push(dtlItem);
            }
        }
        rstArr = newRstArr;
    }
    replaceActDataArr(sourceData, rstArr);
    // fsUtil.writeObjToFile(sourceData.data, 'D:/GitHome/ConstructionCost/tmp/filteredRst.jsp');
}

function preDefineProcess(ctx, tpl, preDefineCfg, rawDataObj) {
    // 依据约定，需要提供如右所示格式的数据：[{field: 'b_code', table: 'mem_stage_bills'}, {field: 'id', table: 'mem_stage_bills'}]
    // 指标对象的mapExpression 格式类似于： "$PROJECT.REPORT.getProperty('mem_stage_im_zl', 'calc_memo')"
    const fields = [];
    if (preDefineCfg.fields) {
        for (const field of preDefineCfg.fields) {
            const tplF = $JE.F(field.ID, tpl);
            const mapStr = tplF[JV.PROP_FIELD_EXP_MAP].split(','); // 如上，用逗号分割
            let start = 0;
            let end = 0;
            let cnt = 0;
            for (let idx = 0; idx < mapStr[1].length; idx++) {
                if (mapStr[1][idx] === '\'') {
                    cnt++;
                    if (cnt === 1) {
                        start = idx;
                    } else {
                        end = idx;
                        break;
                    }
                }
            }
            const codeKey = mapStr[1].slice(start + 1, end);
            // console.log('field codeKey: ' + codeKey);
            const rstF = { field: codeKey, table: tplF.TableName };
            fields.push(rstF);
        }
    }
    switch (preDefineCfg[JV.PROP_HANDLE_TYPE_PRE_DEFINED_KEY]) {
        case 'changeSort' :
            try {
                // 在预定义方式中，小麦处理原始数据，不需要
                data_analyze_util.changeSort.fun(ctx, rawDataObj, fields);
            } catch (ex) {
                //
            }
            break;
        case '' :
            break;
        default:
            break;
    }
}

function adjustData(sourceData, adjustCfg) {
    const rstArr = [];
    for (const item of getActDataArr(sourceData)) {
        if (item._doc) {
            rstArr.push(item._doc);
        } else {
            rstArr.push(item);
        }
    }
    for (const item of adjustCfg[JV.PROP_ADJUST_COLLECTION]) {
        for (const rec of rstArr) {
            if (item[JV.PROP_ADJUST_ACTION] === 'prefix') {
                rec[item.key] = item[JV.PROP_ADJUST_ACTION_VAL] + rec[item.key];
            } else if (item[JV.PROP_ADJUST_ACTION] === 'suffix') {
                rec[item.key] = rec[item.key] + item[JV.PROP_ADJUST_ACTION_VAL];
            }
        }
    }
    replaceActDataArr(sourceData, rstArr);
}

function sortData(sourceData, sortCfg, prjData) {
    let rst = getActDataArr(sourceData);
    const tempRstArr = [];
    const sortType = sortCfg[JV.PROP_SORT_TYPE];
    const srcData = getActDataArr(sourceData);
    for (const item of srcData) {
        if (item._doc) {
            tempRstArr.push(item._doc);
        } else {
            tempRstArr.push(item);
        }
    }
    function private_normal_sort(destArr, sortKeys) {
        destArr.sort(function(a, b) {
            let compRst = 0;
            for (const comp of sortKeys) {
                const reverse = (comp.order === 'ascend') ? 1 : (-1);
                //
                if (a[comp.key] > b[comp.key]) {
                    compRst = reverse;
                    break;
                } else if (a[comp.key] < b[comp.key]) {
                    compRst = -reverse;
                    break;
                }
            }
            return compRst;
        });
    }
    function private_parent_sort(parentArr, parentKeys, childArr, childKeys) {
        const tmpRst = {};
        const rst = [];
        for (const pItem of parentArr) {
            let pKey = 'key';
            for (const key of parentKeys) {
                pKey += '_' + pItem[key];
            }
            tmpRst[pKey] = [];
        }
        for (const cItem of childArr) {
            let cKey = 'key';
            for (const key of childKeys) {
                cKey += '_' + cItem[key];
            }
            if (tmpRst[cKey]) {
                tmpRst[cKey].push(cItem);
            } else {
                // unknown child value! should be filtered!
            }
        }
        // childArr.splice(0);
        for (const pItem of parentArr) {
            let pKey = 'key';
            for (const key of parentKeys) {
                pKey += '_' + pItem[key];
            }
            rst.push(tmpRst[pKey]);
            // for (let rItem of tmpRst[pKey]) {
            //     childArr.push(rItem);
            // }
        }
        return rst;
    }
    switch (sortType) {
        case 'tree':
            const addLevel = true;
            rst = treeUtil.buildTreeNodeDirectly(tempRstArr, addLevel);
            let newTopArr = [];
            if ((sortCfg[JV.PROP_FILTER_TOP_BILLS_NODES] && sortCfg[JV.PROP_FILTER_TOP_BILLS_NODES].length > 0) ||
                (sortCfg[JV.PROP_FILTER_OTHER_BILLS_NODES] && sortCfg[JV.PROP_FILTER_OTHER_BILLS_NODES].length > 0)) {
                const local_check_bills = function(tItem) {
                    let chkDtl = false;
                    if (tItem.flags && tItem.flags.length > 0) {
                        for (const flagItem of tItem.flags) {
                            if (sortCfg[JV.PROP_FILTER_OTHER_BILLS_NODES].indexOf(flagItem.flag) >= 0) {
                                newTopArr.push(tItem);
                                chkDtl = true;
                                break;
                            }
                        }
                    }
                    if (!chkDtl && tItem.items && tItem.items.length > 0) {
                        for (const dtlItem of tItem.items) {
                            local_check_bills(dtlItem);
                        }
                    }
                };
                for (const topItem of rst) {
                    let chkTop = false;
                    if (topItem.flags && topItem.flags.length > 0) {
                        for (const flagItem of topItem.flags) {
                            if (sortCfg[JV.PROP_FILTER_TOP_BILLS_NODES].indexOf(flagItem.flag) >= 0) {
                                newTopArr.push(topItem);
                                chkTop = true;
                                break;
                            }
                        }
                    }
                    if (!chkTop && sortCfg[JV.PROP_FILTER_OTHER_BILLS_NODES] && sortCfg[JV.PROP_FILTER_OTHER_BILLS_NODES].length > 0) {
                        local_check_bills(topItem);
                    }
                }
            } else {
                newTopArr = rst;
            }
            const destArr = [];
            // fsUtil.writeObjToFile(newTopArr, 'D:/GitHome/ConstructionCost/tmp/sortedAndFlattedRstBefore.jsp');
            treeUtil.getFlatArray(newTopArr, destArr, true);
            // console.log(destArr);
            replaceActDataArr(sourceData, destArr);
            // fsUtil.writeObjToFile(sourceData.data, 'D:/GitHome/ConstructionCost/tmp/sortedAndFlattedRst.jsp');
            break;
        case 'normal':
            private_normal_sort(tempRstArr, sortCfg[JV.PROP_SORT_KEYS]);
            replaceActDataArr(sourceData, tempRstArr);
            // fsUtil.writeObjToFile(sourceData.data, 'D:/GitHome/ConstructionCost/tmp/normalSortedRst.jsp');
            break;
        case 'accord_to_parent':
            const pcKey = sortCfg[JV.PROP_PARENT_CHILD_SORT_KEY];
            const parentSrcData = getModuleDataByKey(prjData, pcKey[JV.PROP_PARENT_DATA_KEY]);
            if (parentSrcData) {
                const tempParentArr = [];
                for (const item of getActDataArr(parentSrcData)) {
                    if (item._doc) {
                        tempParentArr.push(item._doc);
                    } else {
                        tempParentArr.push(item);
                    }
                }
                const sortedRstArr = private_parent_sort(tempParentArr, pcKey[JV.PROP_PARENT_SORT_KEYS], tempRstArr, pcKey[JV.PROP_CHILD_SORT_KEYS]);
                if (sortCfg[JV.PROP_OTHER_SUB_SORT] && sortCfg[JV.PROP_OTHER_SUB_SORT].length > 0) {
                    for (const sort of sortCfg[JV.PROP_OTHER_SUB_SORT]) {
                        if (sort[JV.PROP_SORT_TYPE] === 'normal') {
                            for (const subArr of sortedRstArr) {
                                private_normal_sort(subArr, sort[JV.PROP_SORT_KEYS]);
                            }
                        } else if (sort[JV.PROP_SORT_TYPE] === 'self_define') {
                            for (const subArr of sortedRstArr) {
                                // console.log(subArr);
                                const selfDefFunc = null;
                                eval('selfDefFunc = ' + sort[JV.PROP_SORT_TYPE_SELF_DEFINE_LOGIC]);
                                subArr.sort(selfDefFunc);
                                // console.log(subArr);
                            }
                        }
                    }
                }
                tempRstArr.splice(0);
                for (const item of sortedRstArr) {
                    for (const subItem of item) {
                        tempRstArr.push(subItem);
                    }
                }
            }
            replaceActDataArr(sourceData, tempRstArr);
            break;
        case 'self_define':
            if (sortCfg[JV.PROP_SORT_TYPE_SELF_DEFINE_LOGIC]) {
                let selfDefFuncA = null;
                eval('selfDefFuncA = ' + sortCfg[JV.PROP_SORT_TYPE_SELF_DEFINE_LOGIC]);
                if (selfDefFuncA !== null) {
                    tempRstArr.sort(selfDefFuncA);
                } else {
                    console.log('sorting function is null!!!');
                }
            }
            replaceActDataArr(sourceData, tempRstArr);
            break;
        default:
            //
    }
    return rst;
}

function setupFunc(obj, ownRawObj, baseDir) {
    obj.myOwnRawDataObj = ownRawObj;
    obj.baseDir = baseDir;
    obj.getProperty = ext_getProperty;
    obj.getSplitProperty = ext_getSplitProperty;
    obj.getPicProperty = ext_getPicProperty;
    // obj.getPropertyByForeignId = ext_getPropertyByForeignId;
    obj.getArrayProperty = ext_getArrayValues;
    obj.getBlank = ext_getBlank;
}

function assembleFields(fieldList, rstDataArr, $PROJECT) {
    if (fieldList) {
        for (const field of fieldList) {
            shielded_exec_env($PROJECT, field, rstDataArr);
            if ('Precision' in field) {
                if (field.Precision.type === 'fixed') {
                    const vrst = eval(field.Precision.fixedMapExpression);
                    if (vrst && vrst.length === 1) {
                        field.fixedPrecisionNum = vrst[0];
                        vrst.splice(0,1);
                    }
                } else if (field.Precision.type === 'flexible') {
                    const vrst = eval(field.Precision.flexibleMapExpression);
                    if (vrst && vrst.length === 1) {
                        field.flexiblePrecisionRefObj = vrst[0];
                        vrst.splice(0, 1);
                    }
                }
            }
        }
    }
}

function shielded_exec_env($PROJECT, $ME, rptDataItemObj) {
    if ($ME[JV.PROP_FIELD_EXP_MAP]) {
        rptDataItemObj.push(eval($ME[JV.PROP_FIELD_EXP_MAP]));
    }
}

function getActPropertyVal(firstPropKey, secPropKey, orgObj) {
    let rst = null;
    if (orgObj[firstPropKey]) {
        rst = orgObj[firstPropKey];
    } else if (orgObj[secPropKey]) {
        rst = orgObj[secPropKey];
    }
    return rst;
}

function getDeepProperty(propKey, orgObj, destArr) {
    const keys = propKey.split('.');
    const dftPropKey = 'key';
    const dftPropVal = 'value';
    const secDftPropVal = 'items';
    let parent = orgObj;
    let lastVal = null;
    for (const key of keys) {
        if (parent instanceof Array) {
            for (const item of parent) {
                if (item[dftPropKey] === key) {
                    lastVal = getActPropertyVal(dftPropVal, secDftPropVal, item);
                    break;
                }
            }
        } else {
            lastVal = null;
            if (parent[key] !== undefined) {
                lastVal = parent[key];
            } else if (parent[secDftPropVal]) {
                for (const item of parent[secDftPropVal]) {
                    if (item[dftPropKey] === key) {
                        // lastVal = item[dftPropVal];
                        lastVal = getActPropertyVal(dftPropVal, secDftPropVal, item);
                        break;
                    }
                }
            }
        }
        parent = lastVal;
        if (parent === null) break;
    }
    if (destArr && destArr instanceof Array) {
        destArr.push(lastVal);
    }
}

function ext_getProperty(dataKey, propKey) {
    const rst = [];
    const parentObj = this;
    const dtObj = parentObj.myOwnRawDataObj[dataKey];
    // console.log('dataKey: ' + dataKey);
    // console.log(dtObj);
    if (propKey && dtObj) {
        // console.log('---- dtObj[' + dataKey + '] ----');
        // console.log(dtObj[dataKey]);
        const da = getActDataArr(dtObj);
        // console.log(da);
        for (const dItem of da) {
            // const doc = (dItem._doc === null || dItem._doc === undefined) ? dItem : dItem._doc;
            pri_push_property(propKey, dItem, rst);
        }
    }
    // console.log('---- result ----');
    // console.log(rst);
    return rst;
}

function ext_getSplitProperty(dataKey, propKey, splitChar, index, dftValue) {
    const rst = [];
    const parentObj = this;
    const dtObj = parentObj.myOwnRawDataObj[dataKey];
    if (propKey && dtObj) {
        const da = getActDataArr(dtObj);
        for (const dItem of da) {
            pri_push_property(propKey, dItem, rst);
        }
    }
    // const rst = ext_getProperty(dataKey, propKey);
    for (let idx = 0; idx < rst.length; idx++) {
        if (typeof rst[idx] === 'string') {
            const splitArr = rst[idx].split(splitChar);
            if (splitArr.length > index) {
                rst[idx] = splitArr[index];
            } else {
                rst[idx] = dftValue;
            }
        }
    }
    return rst;
}

async function ext_getPicProperty(dataKey, propKey, isPath) {
    const rst = [];
    const parentObj = this;
    const dtObj = parentObj.myOwnRawDataObj[dataKey];
    if (propKey && dtObj) {
        const da = getActDataArr(dtObj);
        for (const dItem of da) {
            pri_push_property(propKey, dItem, rst);
        }
        if (isPath) {
            for (let picIdx = 0; picIdx < rst.length; picIdx++) {
                if (rst[picIdx] !== undefined && rst[picIdx] !== null && rst[picIdx] !== '') {
                    const filePath = parentObj.baseDir + '/app/' + rst[picIdx];
                    try {
                        fs.accessSync(filePath, fs.constants.R_OK);
                        const bData = fs.readFileSync(filePath);
                        const base64Str = bData.toString('base64');
                        const datauri = 'data:image/png;base64,' + base64Str;
                        rst[picIdx] = datauri;
                        // const res = await isFileExisted(filePath);
                        // if (res) {
                        //     const bData = fs.readFileSync(filePath);
                        //     const base64Str = bData.toString('base64');
                        //     const datauri = 'data:image/png;base64,' + base64Str;
                        //     rst[picIdx] = datauri;
                        // } else {
                        //     rst[picIdx] = ''; // 不存在图片，则设置为空，后续的方法就不会当作图片来处理了
                        // }
                    } catch (err) {
                        rst[picIdx] = ''; // 不存在图片，则设置为空，后续的方法就不会当作图片来处理了
                        console.error(err);
                    }
                }
            }
        } else {
            // 不是路径，那么必须是image data，不用处理，正常走即可
        }
    }
    return rst;
}

function ext_getArrayValues(dataKey, itemKey) {
    const rst = [];
    const parentObj = this;
    const dtObj = parentObj.myOwnRawDataObj[dataKey];
    // 计量需要重写
    const keysArr = itemKey.split('.');
    const da = getActDataArr(dtObj);
    // console.log(keysArr);
    for (const dataItem of da) {
        let itemArr = [];
        if (keysArr.length <= 2) {
            if (dataItem[keysArr[0]] instanceof Array) {
                if (keysArr.length === 2) {
                    for (const item of dataItem[keysArr[0]]) {
                        itemArr.push(item[keysArr[1]]);
                    }
                } else {
                    itemArr = itemArr.concat(dataItem[keysArr[0]]);
                }
            } else {
                if (keysArr.length === 2) {
                    const subProperty = dataItem[keysArr[0]][keysArr[1]];
                    if (subProperty instanceof Array) {
                        itemArr = itemArr.concat(subProperty);
                    } else {
                        itemArr.push(subProperty);
                    }
                } else {
                    itemArr.push(dataItem[keysArr[0]]);
                }
            }
        }
        rst.push(itemArr);
        // console.log(itemArr);
    }
    return rst;
}

function ext_getBlank(dataKey, dftVal) {
    const rst = [];
    const parentObj = this;
    const dtObj = parentObj.myOwnRawDataObj[dataKey]; // dataKey是有必要的，必须知道是哪个Table
    // 计量需要重写
    if (dtObj) {
        const dtData = getActDataArr(dtObj);
        for (let i = 0; i < dtData.length; i++) {
            if (dftVal !== null && dftVal !== undefined) {
                rst.push(dftVal)
            } else rst.push('');
        }
    }
    return rst;
}

function ext_getPropertyByForeignId(foreignIdVal, adHocIdKey, propKey, dftValIfNotFound) {
    const rst = [];
    // 计量需要重写
    return rst;
}

function getActDataArr(dtObj) {
    let rst = [];
    if (dtObj) {
        if (dtObj instanceof Array) {
            rst = rst.concat(dtObj);
        } else {
            rst.push(dtObj);
        }
    }
    return rst;
}

function replaceActDataArr(dtObj, newArr) {
    if (dtObj.moduleName === 'projectGLJ') {
        delete dtObj.data.gljList;
        dtObj.data.gljList = newArr;
    } else {
        delete dtObj.data;
        dtObj.data = newArr;
    }
}

function pri_push_property(propKey, doc, rst) {
    if (propKey instanceof Array) {
        // 备注：这里的key数组表示取value的优先级
        for (let pi = 0; pi < propKey.length; pi++) {
            if (doc.hasOwnProperty('property')) {
                if (doc['property'].hasOwnProperty(propKey[pi])) {
                    rst.push(doc['property'][propKey[pi]]);
                    break;
                }
            } else if (doc.hasOwnProperty(propKey[pi])) {
                rst.push(doc[propKey[pi]]);
                break;
            } else {
                const lenBefore = rst.length;
                getDeepProperty(propKey[pi], doc, rst);
                if (rst.length === (lenBefore + 1)) {
                    if (rst[lenBefore] !== null && rst[lenBefore] !== undefined && rst[lenBefore] !== '') {
                        break;
                    } else {
                        rst.splice(-1, 1); // 删除末尾一条数据，给后面留空间
                    }
                }
            }
            if (pi === propKey.length - 1) rst.push('');
        }
    } else {
        if (doc.hasOwnProperty('property') && doc['property'].hasOwnProperty(propKey)) {
            rst.push(doc['property'][propKey]);
        } else if (doc.hasOwnProperty(propKey)) {
            rst.push(doc[propKey]);
        } else {
            getDeepProperty(propKey, doc, rst);
        }
    }
}

// export default Rpt_Data_Extractor;
module.exports = Rpt_Data_Extractor;