update js
This commit is contained in:
HSunboy
467
lib/scripts/generateParsers.js
Normal file
467
lib/scripts/generateParsers.js
Normal file
@ -0,0 +1,467 @@
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const exec = require('child_process').exec;
|
||||
const LICENSE = '// Licensed to Cloudera, Inc. under one\n' +
|
||||
'// or more contributor license agreements. See the NOTICE file\n' +
|
||||
'// distributed with this work for additional information\n' +
|
||||
'// regarding copyright ownership. Cloudera, Inc. licenses this file\n' +
|
||||
'// to you under the Apache License, Version 2.0 (the\n' +
|
||||
'// "License"); you may not use this file except in compliance\n' +
|
||||
'// with the License. You may obtain a copy of the License at\n' +
|
||||
'//\n' +
|
||||
'// http://www.apache.org/licenses/LICENSE-2.0\n' +
|
||||
'//\n' +
|
||||
'// Unless required by applicable law or agreed to in writing, software\n' +
|
||||
'// distributed under the License is distributed on an "AS IS" BASIS,\n' +
|
||||
'// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n' +
|
||||
'// See the License for the specific language governing permissions and\n' +
|
||||
'// limitations under the License.\n';
|
||||
const SQL_STATEMENTS_PARSER_JSDOC = '/**\n' +
|
||||
' * @param {string} input\n' +
|
||||
' *\n' +
|
||||
' * @return {SqlStatementsParserResult}\n' +
|
||||
' */\n';
|
||||
const PARSER_FOLDER = path.join(process.cwd(), 'src/core/parse/');
|
||||
const JISON_FOLDER = path.join(process.cwd(), 'src/jison/');
|
||||
const SQL_PARSER_REPOSITORY_PATH = path.join(PARSER_FOLDER, 'sqlParserRepository.js');
|
||||
const SYNTAX_PARSER_IMPORT_TEMPLATE = ' KEY: require("KEY/KEYSyntaxParser")';
|
||||
const AUTOCOMPLETE_PARSER_IMPORT_TEMPLATE = ' KEY: require("KEY/KEYAutocompleteParser")';
|
||||
const parserDefinitions = {
|
||||
globalSearchParser: {
|
||||
sources: [path.join(JISON_FOLDER, 'globalSearchParser.jison')],
|
||||
target: path.join(JISON_FOLDER, 'globalSearchParser.jison'),
|
||||
outputFolder: PARSER_FOLDER,
|
||||
afterParse: contents => new Promise(resolve => {
|
||||
resolve(LICENSE +
|
||||
contents.replace('var globalSearchParser = ', "import SqlParseSupport from './sqlParseSupport';\n\nvar globalSearchParser = ") +
|
||||
'\nexport default globalSearchParser;\n');
|
||||
})
|
||||
},
|
||||
solrFormulaParser: {
|
||||
sources: [path.join(JISON_FOLDER, 'solrFormulaParser.jison')],
|
||||
target: path.join(JISON_FOLDER, 'solrFormulaParser.jison'),
|
||||
outputFolder: PARSER_FOLDER,
|
||||
afterParse: contents => new Promise(resolve => {
|
||||
resolve(LICENSE + contents + 'export default solrFormulaParser;\n');
|
||||
})
|
||||
},
|
||||
solrQueryParser: {
|
||||
sources: [path.join(JISON_FOLDER, 'solrQueryParser.jison')],
|
||||
target: path.join(JISON_FOLDER, 'solrQueryParser.jison'),
|
||||
outputFolder: PARSER_FOLDER,
|
||||
afterParse: contents => new Promise(resolve => {
|
||||
resolve(LICENSE + contents + 'export default solrQueryParser;\n');
|
||||
})
|
||||
},
|
||||
sqlStatementsParser: {
|
||||
sources: [path.join(JISON_FOLDER, 'sqlStatementsParser.jison')],
|
||||
target: path.join(JISON_FOLDER, 'sqlStatementsParser.jison'),
|
||||
outputFolder: PARSER_FOLDER,
|
||||
afterParse: contents => new Promise(resolve => {
|
||||
resolve(LICENSE +
|
||||
contents.replace('parse: function parse', SQL_STATEMENTS_PARSER_JSDOC + 'parse: function parse') +
|
||||
'export default sqlStatementsParser;\n');
|
||||
})
|
||||
}
|
||||
};
|
||||
const mkdir = path => new Promise((resolve, reject) => {
|
||||
if (fs.existsSync(path)) {
|
||||
resolve();
|
||||
}
|
||||
else {
|
||||
fs.mkdir(path, err => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
}
|
||||
});
|
||||
const readFile = path => new Promise((resolve, reject) => {
|
||||
fs.readFile(path, (err, buf) => {
|
||||
if (err) {
|
||||
reject(err);
|
||||
}
|
||||
resolve(buf ? buf.toString() : '');
|
||||
});
|
||||
});
|
||||
const writeFile = (path, contents) => new Promise((resolve, reject) => {
|
||||
fs.writeFile(path, contents, err => {
|
||||
if (err) {
|
||||
reject();
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
const copyFile = (source, destination, contentsCallback) => new Promise((resolve, reject) => {
|
||||
readFile(source)
|
||||
.then(contents => {
|
||||
writeFile(destination, contentsCallback ? contentsCallback(contents) : contents)
|
||||
.then(resolve)
|
||||
.catch(reject);
|
||||
})
|
||||
.catch(reject);
|
||||
});
|
||||
const deleteFile = path => {
|
||||
fs.unlinkSync(path);
|
||||
};
|
||||
const execCmd = cmd => new Promise((resolve, reject) => {
|
||||
exec(cmd, (err, stdout, stderr) => {
|
||||
if (err) {
|
||||
reject('stderr:\n' + stderr + '\n\nstdout:\n' + stdout);
|
||||
}
|
||||
resolve(stdout);
|
||||
});
|
||||
});
|
||||
const generateParser = parserName => new Promise((resolve, reject) => {
|
||||
const parserConfig = parserDefinitions[parserName];
|
||||
/**
|
||||
* 合并jison文件，生成待编译文件
|
||||
*/
|
||||
const concatPromise = new Promise((resolve, reject) => {
|
||||
if (parserConfig.sources.length > 1 && parserConfig.target) {
|
||||
console.log('Concatenating files...');
|
||||
const promises = parserConfig.sources.map(fileName => readFile(fileName));
|
||||
Promise.all(promises)
|
||||
.then(contents => {
|
||||
writeFile(parserConfig.target, contents.join('')).then(() => {
|
||||
resolve(parserConfig.target);
|
||||
});
|
||||
})
|
||||
.catch(reject);
|
||||
}
|
||||
else if (parserConfig.sources.length === 1) {
|
||||
resolve(parserConfig.sources[0]);
|
||||
}
|
||||
else {
|
||||
reject('No jison source specified');
|
||||
}
|
||||
});
|
||||
concatPromise
|
||||
.then(targetPath => {
|
||||
console.log(`Generate precomplier jison success（${targetPath}）...`);
|
||||
let jisonCommand = 'jison ' + targetPath;
|
||||
if (parserConfig.lexer) {
|
||||
jisonCommand += ' ' + parserConfig.lexer;
|
||||
}
|
||||
jisonCommand += ' -m js';
|
||||
console.log('Generating parser...');
|
||||
execCmd(jisonCommand)
|
||||
.then(stdout => {
|
||||
if (/\S/.test(stdout)) {
|
||||
console.log('got output for: ' + jisonCommand);
|
||||
console.log(stdout);
|
||||
}
|
||||
if (parserConfig.sources.length > 1) {
|
||||
deleteFile(targetPath); // Remove concatenated file
|
||||
}
|
||||
console.log('Adjusting JS...');
|
||||
/**
|
||||
* 删除生成文件，复制到配置的文件夹中
|
||||
*/
|
||||
const generatedJsFileName = parserConfig.target
|
||||
.replace('.jison', '.js')
|
||||
.replace(/^.*\/([^/]+)$/, '$1');
|
||||
readFile(generatedJsFileName)
|
||||
.then(contents => {
|
||||
parserConfig
|
||||
.afterParse(contents)
|
||||
.then(finalContents => {
|
||||
writeFile(path.join(parserConfig.outputFolder, generatedJsFileName), finalContents)
|
||||
.then(() => {
|
||||
deleteFile(generatedJsFileName);
|
||||
resolve();
|
||||
})
|
||||
.catch(reject);
|
||||
})
|
||||
.catch(reject);
|
||||
})
|
||||
.catch(reject);
|
||||
})
|
||||
.catch(reject);
|
||||
})
|
||||
.catch(reject);
|
||||
});
|
||||
let parsersToGenerate = [];
|
||||
const invalid = [];
|
||||
let all = false;
|
||||
const listDir = folder => new Promise(resolve => {
|
||||
fs.readdir(folder, (err, files) => {
|
||||
resolve(files);
|
||||
});
|
||||
});
|
||||
/**
|
||||
* 构造，添加子语言模块编译配置
|
||||
* @param {*} fileIndex 文件的存在表
|
||||
* @param {*} folder 对应的子语言文件夹
|
||||
* @param {*} sharedFiles 子语言核心jison文件
|
||||
* @param {*} autocomplete 是否为补全文件
|
||||
*/
|
||||
const findParser = (fileIndex, folder, sharedFiles, autocomplete) => {
|
||||
const prefix = autocomplete ? 'autocomplete' : 'syntax';
|
||||
if (fileIndex[prefix + '_header.jison'] && fileIndex[prefix + '_footer.jison']) {
|
||||
const parserName = folder + (autocomplete ? 'AutocompleteParser' : 'SyntaxParser');
|
||||
const parserDefinition = {
|
||||
sources: [path.join(JISON_FOLDER, 'sql', folder, prefix + '_header.jison')].concat(sharedFiles),
|
||||
lexer: path.join(JISON_FOLDER, 'sql', folder, '/sql.jisonlex'),
|
||||
target: path.join(JISON_FOLDER, 'sql', folder, parserName + '.jison'),
|
||||
sqlParser: autocomplete ? 'AUTOCOMPLETE' : 'SYNTAX',
|
||||
outputFolder: path.join(PARSER_FOLDER, folder),
|
||||
afterParse: contents => new Promise(resolve => {
|
||||
resolve(LICENSE +
|
||||
contents
|
||||
.replace('var ' + parserName + ' = ', "import SqlParseSupport from " +
|
||||
"'./sqlParseSupport';\n\nvar " +
|
||||
parserName +
|
||||
' = ')
|
||||
.replace('loc: yyloc,', "loc: lexer.yylloc, ruleId: stack.slice(stack.length - 2, stack.length).join(''),") +
|
||||
'\nexport default ' +
|
||||
parserName +
|
||||
';\n');
|
||||
})
|
||||
};
|
||||
parserDefinition.sources.push(path.join(JISON_FOLDER, 'sql', folder, prefix + '_footer.jison'));
|
||||
parserDefinitions[parserName] = parserDefinition;
|
||||
}
|
||||
else {
|
||||
console.log("Warn: Could not find '" +
|
||||
prefix +
|
||||
"_header.jison' or '" +
|
||||
prefix +
|
||||
"_footer.jison' in " +
|
||||
JISON_FOLDER +
|
||||
'sql/' +
|
||||
folder +
|
||||
'/');
|
||||
}
|
||||
};
|
||||
/**
|
||||
* 添加所有子语言编译配置
|
||||
*/
|
||||
const identifySqlParsers = () => new Promise(resolve => {
|
||||
listDir(JISON_FOLDER + 'sql').then(files => {
|
||||
const promises = [];
|
||||
files.forEach(folder => {
|
||||
const subLanguageJisonFolder = path.join(JISON_FOLDER, 'sql', folder);
|
||||
promises.push(
|
||||
/**
|
||||
* 遍历具体的语言目录
|
||||
*/
|
||||
listDir(subLanguageJisonFolder).then(jisonFiles => {
|
||||
/**
|
||||
* 文件目录记录表
|
||||
*/
|
||||
const fileIndex = {};
|
||||
jisonFiles.forEach(jisonFile => {
|
||||
fileIndex[jisonFile] = true;
|
||||
});
|
||||
/**
|
||||
* 挑选核心的jison文件（剥除autocomplate，syntax的功能文件）
|
||||
*/
|
||||
const sharedFiles = jisonFiles
|
||||
.filter(jisonFile => jisonFile.indexOf('sql_') !== -1)
|
||||
.map(jisonFile => path.join(subLanguageJisonFolder, jisonFile));
|
||||
if (fileIndex['sql.jisonlex']) {
|
||||
/**
|
||||
* 添加子语言自动补全编译配置
|
||||
* 加入了error.jison，为了在校验失败的情况下也能够提示？
|
||||
*/
|
||||
findParser(fileIndex, folder, sharedFiles, true);
|
||||
/**
|
||||
* 添加子语言语法检查配置
|
||||
*/
|
||||
findParser(fileIndex, folder, sharedFiles.filter(path => path.indexOf('_error.jison') === -1), false);
|
||||
}
|
||||
else {
|
||||
console.log("Warn: Could not find 'sql.jisonlex' in " + JISON_FOLDER + 'sql/' + folder + '/');
|
||||
}
|
||||
}));
|
||||
});
|
||||
Promise.all(promises).then(resolve);
|
||||
});
|
||||
});
|
||||
const copyTests = (source, target) => new Promise((resolve, reject) => {
|
||||
const replaceRegexp = new RegExp(source + '(Autocomplete|Syntax)Parser', 'g');
|
||||
mkdir(PARSER_FOLDER + target)
|
||||
.then(() => {
|
||||
mkdir(PARSER_FOLDER + target + '/test')
|
||||
.then(() => {
|
||||
listDir(PARSER_FOLDER + source + '/test')
|
||||
.then(testFiles => {
|
||||
const copyPromises = [];
|
||||
testFiles.forEach(testFile => {
|
||||
copyPromises.push(copyFile(PARSER_FOLDER + source + '/test/' + testFile, PARSER_FOLDER + target + '/test/' + testFile.replace(source, target), contents => contents.replace(replaceRegexp, target + '$1Parser')));
|
||||
});
|
||||
Promise.all(copyPromises)
|
||||
.then(resolve)
|
||||
.catch(reject);
|
||||
})
|
||||
.catch(reject);
|
||||
})
|
||||
.catch(reject);
|
||||
})
|
||||
.catch(reject);
|
||||
});
|
||||
/**
|
||||
* 校验，配置自定义语言
|
||||
*/
|
||||
const prepareForNewParser = () => new Promise((resolve, reject) => {
|
||||
/**
|
||||
* 根据一个子语言文件夹来生成一个特殊sql名字的语法文件
|
||||
* -new generic postgresql
|
||||
* 根据generic文件夹生成postgresql语法文件
|
||||
*/
|
||||
if (process.argv.length === 3 && process.argv[0] === '-new') {
|
||||
process.argv.shift();
|
||||
const source = process.argv.shift();
|
||||
const target = process.argv.shift();
|
||||
console.log("Generating new parser '" + target + "' based on '" + source + "'...");
|
||||
process.argv.push(target);
|
||||
if (!Object.keys(parserDefinitions).some(key => {
|
||||
if (key.indexOf(source) === 0) {
|
||||
copyTests(source, target)
|
||||
.then(() => {
|
||||
mkdir(JISON_FOLDER + 'sql/' + target)
|
||||
.then(() => {
|
||||
listDir(JISON_FOLDER + 'sql/' + source).then(files => {
|
||||
const copyPromises = [];
|
||||
files.forEach(file => {
|
||||
copyPromises.push(copyFile(JISON_FOLDER + 'sql/' + source + '/' + file, JISON_FOLDER + 'sql/' + target + '/' + file));
|
||||
});
|
||||
Promise.all(copyPromises).then(() => {
|
||||
const autocompleteSources = [
|
||||
'sql/' + target + '/autocomplete_header.jison'
|
||||
];
|
||||
const syntaxSources = ['sql/' + target + '/syntax_header.jison'];
|
||||
files.forEach(file => {
|
||||
if (file.indexOf('sql_') === 0) {
|
||||
autocompleteSources.push('sql/' + target + '/' + file);
|
||||
syntaxSources.push('sql/' + target + '/' + file);
|
||||
}
|
||||
});
|
||||
autocompleteSources.push('sql/' + target + '/autocomplete_footer.jison');
|
||||
syntaxSources.push('sql/' + target + '/syntax_footer.jison');
|
||||
mkdir('desktop/core/src/desktop/js/parse/sql/' + target).then(() => {
|
||||
copyFile('desktop/core/src/desktop/js/parse/sql/' +
|
||||
source +
|
||||
'/sqlParseSupport.js', 'desktop/core/src/desktop/js/parse/sql/' +
|
||||
target +
|
||||
'/sqlParseSupport.js', contents => contents.replace(/parser\.yy\.activeDialect = '[^']+';'/g, "parser.yy.activeDialect = '" + target + "';")).then(() => {
|
||||
identifySqlParsers()
|
||||
.then(resolve)
|
||||
.catch(reject);
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
})
|
||||
.catch(err => {
|
||||
console.log(err);
|
||||
});
|
||||
})
|
||||
.catch(reject);
|
||||
return true;
|
||||
}
|
||||
})) {
|
||||
reject("No existing parser found for '" + source + "'");
|
||||
}
|
||||
}
|
||||
else {
|
||||
resolve();
|
||||
}
|
||||
});
|
||||
identifySqlParsers().then(() => {
|
||||
process.argv.shift();
|
||||
process.argv.shift();
|
||||
console.log('Generate sub language success...');
|
||||
prepareForNewParser().then(() => {
|
||||
console.log('Generate custom language success...');
|
||||
process.argv.forEach(arg => {
|
||||
if (arg === 'all') {
|
||||
/**
|
||||
* 编译全部
|
||||
*/
|
||||
all = true;
|
||||
}
|
||||
else if (parserDefinitions[arg]) {
|
||||
/**
|
||||
* 特点编译目标
|
||||
*/
|
||||
parsersToGenerate.push(arg);
|
||||
}
|
||||
else {
|
||||
/**
|
||||
* 根据关键字匹配编译目标
|
||||
*/
|
||||
let prefixFound = false;
|
||||
Object.keys(parserDefinitions).forEach(key => {
|
||||
if (key.indexOf(arg) === 0) {
|
||||
prefixFound = true;
|
||||
parsersToGenerate.push(key);
|
||||
}
|
||||
});
|
||||
if (!prefixFound) {
|
||||
invalid.push(arg);
|
||||
}
|
||||
}
|
||||
});
|
||||
if (all) {
|
||||
parsersToGenerate = Object.keys(parserDefinitions);
|
||||
}
|
||||
if (invalid.length) {
|
||||
console.log("No parser config found for: '" + invalid.join("', '") + "'");
|
||||
console.log('\nPossible options are:\n ' +
|
||||
['all'].concat(Object.keys(parserDefinitions)).join('\n ') +
|
||||
'\n');
|
||||
return;
|
||||
}
|
||||
const parserCount = parsersToGenerate.length;
|
||||
let idx = 0;
|
||||
/**
|
||||
* 执行编译
|
||||
*/
|
||||
const generateRecursive = () => {
|
||||
idx++;
|
||||
if (parsersToGenerate.length) {
|
||||
const parserName = parsersToGenerate.pop();
|
||||
if (parserCount > 1) {
|
||||
console.log("Generating '" + parserName + "' (" + idx + '/' + parserCount + ')...');
|
||||
}
|
||||
else {
|
||||
console.log("Generating '" + parserName + "'...");
|
||||
}
|
||||
generateParser(parserName)
|
||||
.then(generateRecursive)
|
||||
.catch(error => {
|
||||
console.log(error);
|
||||
console.log('FAIL!');
|
||||
});
|
||||
}
|
||||
else {
|
||||
const autocompParsers = [];
|
||||
const syntaxParsers = [];
|
||||
console.log('Updating sqlParserRepository.js...');
|
||||
Object.keys(parserDefinitions).forEach(key => {
|
||||
if (parserDefinitions[key].sqlParser === 'AUTOCOMPLETE') {
|
||||
autocompParsers.push(AUTOCOMPLETE_PARSER_IMPORT_TEMPLATE.replace(/KEY/g, key.replace('AutocompleteParser', '')));
|
||||
}
|
||||
else if (parserDefinitions[key].sqlParser === 'SYNTAX') {
|
||||
syntaxParsers.push(SYNTAX_PARSER_IMPORT_TEMPLATE.replace(/KEY/g, key.replace('SyntaxParser', '')));
|
||||
}
|
||||
});
|
||||
readFile(SQL_PARSER_REPOSITORY_PATH).then(contents => {
|
||||
contents = contents.replace(/const SYNTAX_MODULES = [^}]+}/, 'const SYNTAX_MODULES = {\n' + syntaxParsers.sort().join(',\n') + '\n}');
|
||||
contents = contents.replace(/const AUTOCOMPLETE_MODULES = [^}]+}/, 'const AUTOCOMPLETE_MODULES = {\n' + autocompParsers.sort().join(',\n') + '\n}');
|
||||
writeFile(SQL_PARSER_REPOSITORY_PATH, contents).then(() => {
|
||||
console.log('Done!\n');
|
||||
});
|
||||
});
|
||||
}
|
||||
};
|
||||
/**
|
||||
* 集中精力办大事
|
||||
*/
|
||||
generateRecursive();
|
||||
});
|
||||
});
|
||||
/* eslint-enable no-restricted-syntax */
|
||||
Reference in New Issue
Block a user