const fs = require('fs'); const path = require('path'); const exec = require('child_process').exec; const LICENSE = '// Licensed to Cloudera, Inc. under one\n' + '// or more contributor license agreements. See the NOTICE file\n' + '// distributed with this work for additional information\n' + '// regarding copyright ownership. Cloudera, Inc. licenses this file\n' + '// to you under the Apache License, Version 2.0 (the\n' + '// "License"); you may not use this file except in compliance\n' + '// with the License. You may obtain a copy of the License at\n' + '//\n' + '// http://www.apache.org/licenses/LICENSE-2.0\n' + '//\n' + '// Unless required by applicable law or agreed to in writing, software\n' + '// distributed under the License is distributed on an "AS IS" BASIS,\n' + '// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n' + '// See the License for the specific language governing permissions and\n' + '// limitations under the License.\n'; const SQL_STATEMENTS_PARSER_JSDOC = '/**\n' + ' * @param {string} input\n' + ' *\n' + ' * @return {SqlStatementsParserResult}\n' + ' */\n'; const PARSER_FOLDER = path.join(process.cwd(), 'src/core/parse/'); const JISON_FOLDER = path.join(process.cwd(), 'src/jison/'); const SQL_PARSER_REPOSITORY_PATH = path.join(PARSER_FOLDER, 'sqlParserRepository.js'); const SYNTAX_PARSER_IMPORT_TEMPLATE = ' KEY: require("KEY/KEYSyntaxParser")'; const AUTOCOMPLETE_PARSER_IMPORT_TEMPLATE = ' KEY: require("KEY/KEYAutocompleteParser")'; const parserDefinitions = { globalSearchParser: { sources: [path.join(JISON_FOLDER, 'globalSearchParser.jison')], target: path.join(JISON_FOLDER, 'globalSearchParser.jison'), outputFolder: PARSER_FOLDER, afterParse: contents => new Promise(resolve => { resolve(LICENSE + contents.replace('var globalSearchParser = ', "import SqlParseSupport from './sqlParseSupport';\n\nvar globalSearchParser = ") + '\nexport default globalSearchParser;\n'); }) }, solrFormulaParser: { sources: [path.join(JISON_FOLDER, 'solrFormulaParser.jison')], target: path.join(JISON_FOLDER, 'solrFormulaParser.jison'), outputFolder: PARSER_FOLDER, afterParse: contents => new Promise(resolve => { resolve(LICENSE + contents + 'export default solrFormulaParser;\n'); }) }, solrQueryParser: { sources: [path.join(JISON_FOLDER, 'solrQueryParser.jison')], target: path.join(JISON_FOLDER, 'solrQueryParser.jison'), outputFolder: PARSER_FOLDER, afterParse: contents => new Promise(resolve => { resolve(LICENSE + contents + 'export default solrQueryParser;\n'); }) }, sqlStatementsParser: { sources: [path.join(JISON_FOLDER, 'sqlStatementsParser.jison')], target: path.join(JISON_FOLDER, 'sqlStatementsParser.jison'), outputFolder: PARSER_FOLDER, afterParse: contents => new Promise(resolve => { resolve(LICENSE + contents.replace('parse: function parse', SQL_STATEMENTS_PARSER_JSDOC + 'parse: function parse') + 'export default sqlStatementsParser;\n'); }) } }; const mkdir = path => new Promise((resolve, reject) => { if (fs.existsSync(path)) { resolve(); } else { fs.mkdir(path, err => { if (err) { reject(err); } resolve(); }); } }); const readFile = path => new Promise((resolve, reject) => { fs.readFile(path, (err, buf) => { if (err) { reject(err); } resolve(buf ? buf.toString() : ''); }); }); const writeFile = (path, contents) => new Promise((resolve, reject) => { fs.writeFile(path, contents, err => { if (err) { reject(); } resolve(); }); }); const copyFile = (source, destination, contentsCallback) => new Promise((resolve, reject) => { readFile(source) .then(contents => { writeFile(destination, contentsCallback ? contentsCallback(contents) : contents) .then(resolve) .catch(reject); }) .catch(reject); }); const deleteFile = path => { fs.unlinkSync(path); }; const execCmd = cmd => new Promise((resolve, reject) => { exec(cmd, (err, stdout, stderr) => { if (err) { reject('stderr:\n' + stderr + '\n\nstdout:\n' + stdout); } resolve(stdout); }); }); const generateParser = parserName => new Promise((resolve, reject) => { const parserConfig = parserDefinitions[parserName]; /** * 合并jison文件,生成待编译文件 */ const concatPromise = new Promise((resolve, reject) => { if (parserConfig.sources.length > 1 && parserConfig.target) { console.log('Concatenating files...'); const promises = parserConfig.sources.map(fileName => readFile(fileName)); Promise.all(promises) .then(contents => { writeFile(parserConfig.target, contents.join('')).then(() => { resolve(parserConfig.target); }); }) .catch(reject); } else if (parserConfig.sources.length === 1) { resolve(parserConfig.sources[0]); } else { reject('No jison source specified'); } }); concatPromise .then(targetPath => { console.log(`Generate precomplier jison success(${targetPath})...`); let jisonCommand = 'jison ' + targetPath; if (parserConfig.lexer) { jisonCommand += ' ' + parserConfig.lexer; } jisonCommand += ' -m js'; console.log('Generating parser...'); execCmd(jisonCommand) .then(stdout => { if (/\S/.test(stdout)) { console.log('got output for: ' + jisonCommand); console.log(stdout); } if (parserConfig.sources.length > 1) { deleteFile(targetPath); // Remove concatenated file } console.log('Adjusting JS...'); /** * 删除生成文件,复制到配置的文件夹中 */ const generatedJsFileName = parserConfig.target .replace('.jison', '.js') .replace(/^.*\/([^/]+)$/, '$1'); readFile(generatedJsFileName) .then(contents => { parserConfig .afterParse(contents) .then(finalContents => { writeFile(path.join(parserConfig.outputFolder, generatedJsFileName), finalContents) .then(() => { deleteFile(generatedJsFileName); resolve(); }) .catch(reject); }) .catch(reject); }) .catch(reject); }) .catch(reject); }) .catch(reject); }); let parsersToGenerate = []; const invalid = []; let all = false; const listDir = folder => new Promise(resolve => { fs.readdir(folder, (err, files) => { resolve(files); }); }); /** * 构造,添加子语言模块编译配置 * @param {*} fileIndex 文件的存在表 * @param {*} folder 对应的子语言文件夹 * @param {*} sharedFiles 子语言核心jison文件 * @param {*} autocomplete 是否为补全文件 */ const findParser = (fileIndex, folder, sharedFiles, autocomplete) => { const prefix = autocomplete ? 'autocomplete' : 'syntax'; if (fileIndex[prefix + '_header.jison'] && fileIndex[prefix + '_footer.jison']) { const parserName = folder + (autocomplete ? 'AutocompleteParser' : 'SyntaxParser'); const parserDefinition = { sources: [path.join(JISON_FOLDER, 'sql', folder, prefix + '_header.jison')].concat(sharedFiles), lexer: path.join(JISON_FOLDER, 'sql', folder, '/sql.jisonlex'), target: path.join(JISON_FOLDER, 'sql', folder, parserName + '.jison'), sqlParser: autocomplete ? 'AUTOCOMPLETE' : 'SYNTAX', outputFolder: path.join(PARSER_FOLDER, folder), afterParse: contents => new Promise(resolve => { resolve(LICENSE + contents .replace('var ' + parserName + ' = ', "import SqlParseSupport from " + "'./sqlParseSupport';\n\nvar " + parserName + ' = ') .replace('loc: yyloc,', "loc: lexer.yylloc, ruleId: stack.slice(stack.length - 2, stack.length).join(''),") + '\nexport default ' + parserName + ';\n'); }) }; parserDefinition.sources.push(path.join(JISON_FOLDER, 'sql', folder, prefix + '_footer.jison')); parserDefinitions[parserName] = parserDefinition; } else { console.log("Warn: Could not find '" + prefix + "_header.jison' or '" + prefix + "_footer.jison' in " + JISON_FOLDER + 'sql/' + folder + '/'); } }; /** * 添加所有子语言编译配置 */ const identifySqlParsers = () => new Promise(resolve => { listDir(JISON_FOLDER + 'sql').then(files => { const promises = []; files.forEach(folder => { const subLanguageJisonFolder = path.join(JISON_FOLDER, 'sql', folder); promises.push( /** * 遍历具体的语言目录 */ listDir(subLanguageJisonFolder).then(jisonFiles => { /** * 文件目录记录表 */ const fileIndex = {}; jisonFiles.forEach(jisonFile => { fileIndex[jisonFile] = true; }); /** * 挑选核心的jison文件(剥除autocomplate,syntax的功能文件) */ const sharedFiles = jisonFiles .filter(jisonFile => jisonFile.indexOf('sql_') !== -1) .map(jisonFile => path.join(subLanguageJisonFolder, jisonFile)); if (fileIndex['sql.jisonlex']) { /** * 添加子语言自动补全编译配置 * 加入了error.jison,为了在校验失败的情况下也能够提示? */ findParser(fileIndex, folder, sharedFiles, true); /** * 添加子语言语法检查配置 */ findParser(fileIndex, folder, sharedFiles.filter(path => path.indexOf('_error.jison') === -1), false); } else { console.log("Warn: Could not find 'sql.jisonlex' in " + JISON_FOLDER + 'sql/' + folder + '/'); } })); }); Promise.all(promises).then(resolve); }); }); const copyTests = (source, target) => new Promise((resolve, reject) => { const replaceRegexp = new RegExp(source + '(Autocomplete|Syntax)Parser', 'g'); mkdir(PARSER_FOLDER + target) .then(() => { mkdir(PARSER_FOLDER + target + '/test') .then(() => { listDir(PARSER_FOLDER + source + '/test') .then(testFiles => { const copyPromises = []; testFiles.forEach(testFile => { copyPromises.push(copyFile(PARSER_FOLDER + source + '/test/' + testFile, PARSER_FOLDER + target + '/test/' + testFile.replace(source, target), contents => contents.replace(replaceRegexp, target + '$1Parser'))); }); Promise.all(copyPromises) .then(resolve) .catch(reject); }) .catch(reject); }) .catch(reject); }) .catch(reject); }); /** * 校验,配置自定义语言 */ const prepareForNewParser = () => new Promise((resolve, reject) => { /** * 根据一个子语言文件夹来生成一个特殊sql名字的语法文件 * -new generic postgresql * 根据generic文件夹生成postgresql语法文件 */ if (process.argv.length === 3 && process.argv[0] === '-new') { process.argv.shift(); const source = process.argv.shift(); const target = process.argv.shift(); console.log("Generating new parser '" + target + "' based on '" + source + "'..."); process.argv.push(target); if (!Object.keys(parserDefinitions).some(key => { if (key.indexOf(source) === 0) { copyTests(source, target) .then(() => { mkdir(JISON_FOLDER + 'sql/' + target) .then(() => { listDir(JISON_FOLDER + 'sql/' + source).then(files => { const copyPromises = []; files.forEach(file => { copyPromises.push(copyFile(JISON_FOLDER + 'sql/' + source + '/' + file, JISON_FOLDER + 'sql/' + target + '/' + file)); }); Promise.all(copyPromises).then(() => { const autocompleteSources = [ 'sql/' + target + '/autocomplete_header.jison' ]; const syntaxSources = ['sql/' + target + '/syntax_header.jison']; files.forEach(file => { if (file.indexOf('sql_') === 0) { autocompleteSources.push('sql/' + target + '/' + file); syntaxSources.push('sql/' + target + '/' + file); } }); autocompleteSources.push('sql/' + target + '/autocomplete_footer.jison'); syntaxSources.push('sql/' + target + '/syntax_footer.jison'); mkdir('desktop/core/src/desktop/js/parse/sql/' + target).then(() => { copyFile('desktop/core/src/desktop/js/parse/sql/' + source + '/sqlParseSupport.js', 'desktop/core/src/desktop/js/parse/sql/' + target + '/sqlParseSupport.js', contents => contents.replace(/parser\.yy\.activeDialect = '[^']+';'/g, "parser.yy.activeDialect = '" + target + "';")).then(() => { identifySqlParsers() .then(resolve) .catch(reject); }); }); }); }); }) .catch(err => { console.log(err); }); }) .catch(reject); return true; } })) { reject("No existing parser found for '" + source + "'"); } } else { resolve(); } }); identifySqlParsers().then(() => { process.argv.shift(); process.argv.shift(); console.log('Generate sub language success...'); prepareForNewParser().then(() => { console.log('Generate custom language success...'); process.argv.forEach(arg => { if (arg === 'all') { /** * 编译全部 */ all = true; } else if (parserDefinitions[arg]) { /** * 特点编译目标 */ parsersToGenerate.push(arg); } else { /** * 根据关键字匹配编译目标 */ let prefixFound = false; Object.keys(parserDefinitions).forEach(key => { if (key.indexOf(arg) === 0) { prefixFound = true; parsersToGenerate.push(key); } }); if (!prefixFound) { invalid.push(arg); } } }); if (all) { parsersToGenerate = Object.keys(parserDefinitions); } if (invalid.length) { console.log("No parser config found for: '" + invalid.join("', '") + "'"); console.log('\nPossible options are:\n ' + ['all'].concat(Object.keys(parserDefinitions)).join('\n ') + '\n'); return; } const parserCount = parsersToGenerate.length; let idx = 0; /** * 执行编译 */ const generateRecursive = () => { idx++; if (parsersToGenerate.length) { const parserName = parsersToGenerate.pop(); if (parserCount > 1) { console.log("Generating '" + parserName + "' (" + idx + '/' + parserCount + ')...'); } else { console.log("Generating '" + parserName + "'..."); } generateParser(parserName) .then(generateRecursive) .catch(error => { console.log(error); console.log('FAIL!'); }); } else { const autocompParsers = []; const syntaxParsers = []; console.log('Updating sqlParserRepository.js...'); Object.keys(parserDefinitions).forEach(key => { if (parserDefinitions[key].sqlParser === 'AUTOCOMPLETE') { autocompParsers.push(AUTOCOMPLETE_PARSER_IMPORT_TEMPLATE.replace(/KEY/g, key.replace('AutocompleteParser', ''))); } else if (parserDefinitions[key].sqlParser === 'SYNTAX') { syntaxParsers.push(SYNTAX_PARSER_IMPORT_TEMPLATE.replace(/KEY/g, key.replace('SyntaxParser', ''))); } }); readFile(SQL_PARSER_REPOSITORY_PATH).then(contents => { contents = contents.replace(/const SYNTAX_MODULES = [^}]+}/, 'const SYNTAX_MODULES = {\n' + syntaxParsers.sort().join(',\n') + '\n}'); contents = contents.replace(/const AUTOCOMPLETE_MODULES = [^}]+}/, 'const AUTOCOMPLETE_MODULES = {\n' + autocompParsers.sort().join(',\n') + '\n}'); writeFile(SQL_PARSER_REPOSITORY_PATH, contents).then(() => { console.log('Done!\n'); }); }); } }; /** * 集中精力办大事 */ generateRecursive(); }); }); /* eslint-enable no-restricted-syntax */