monaco-editor/test/tokenization.html

326 lines
8.2 KiB
HTML

<!DOCTYPE html>
<html>
<head>
<meta http-equiv="X-UA-Compatible" content="IE=edge" />
<meta http-equiv="Content-Type" content="text/html;charset=utf-8" />
<link rel="stylesheet" href="https://code.jquery.com/qunit/qunit-2.0.1.css">
</head>
<body>
<script src="../metadata.js"></script>
<script src="dev-setup.js"></script>
<script src="tokenization.js"></script>
<script src="https://code.jquery.com/qunit/qunit-2.0.1.js"></script>
<script>
var GENERATE_TOKENS = false;
var LIMIT_TO_TEST = null;//'sample - xml';
loadEditor(function() {
let lazyModules = [
'vs/basic-languages/src/bat',
'vs/basic-languages/src/coffee',
'vs/basic-languages/src/cpp',
'vs/basic-languages/src/csharp',
'vs/basic-languages/src/dockerfile',
'vs/basic-languages/src/fsharp',
'vs/basic-languages/src/go',
'vs/basic-languages/src/handlebars',
'vs/basic-languages/src/html',
'vs/basic-languages/src/ini',
'vs/basic-languages/src/pug',
'vs/basic-languages/src/java',
'vs/basic-languages/src/lua',
'vs/basic-languages/src/markdown',
'vs/basic-languages/src/objective-c',
'vs/basic-languages/src/postiats',
'vs/basic-languages/src/php',
'vs/basic-languages/src/powershell',
'vs/basic-languages/src/python',
'vs/basic-languages/src/r',
'vs/basic-languages/src/razor',
'vs/basic-languages/src/ruby',
'vs/basic-languages/src/swift',
'vs/basic-languages/src/sql',
'vs/basic-languages/src/vb',
'vs/basic-languages/src/xml',
'vs/basic-languages/src/less',
'vs/basic-languages/src/scss',
'vs/basic-languages/src/css',
'vs/basic-languages/src/yaml',
'vs/language/typescript/src/mode',
'vs/language/css/cssMode',
'vs/language/json/jsonMode',
'vs/language/html/htmlMode'
];
console.log('editor loaded');
require(lazyModules, function() {
console.log('all lazy code loaded');
function resolveSample(sample) {
return sample.loadText().then(function(txt) {
return {
name: sample.name,
language: sample.mimeType,
text: txt
};
});
}
require(['./samples'], function(SAMPLES) {
monaco.Promise.join(
SAMPLES.map(resolveSample)
).then(function(samples) {
renderSamples(samples);
});
});
});
});
function renderSamples(samples) {
samples = samples.filter(function(sample) {
if (typeof sample.language !== 'string') {
return false;
}
if (sample.language === 'plaintext' || sample.language === 'text/plain') {
return false;
}
// return sample.name === 'sample - css';
return /sample/.test(sample.name);
});
samples.sort(function(a,b) {
if (a.language === b.language) {
if (a.text < b.text) {
return -1;
}
if (a.text > b.text) {
return 1;
}
return 0;
}
if (a.language < b.language) {
return -1;
}
return 1;
});
samples.forEach(renderSample);
generateTests();
}
var TEST_CASES = [];
function renderSample(sample, index) {
console.log('TEST #' + index + ': ' + sample.name);
if (LIMIT_TO_TEST && sample.name !== LIMIT_TO_TEST) {
TEST_CASES.push({ name: sample.name });
return;
}
var testCase = {
name: sample.name,
// text: sample.text,
language: sample.language,
result: {
vs: extractColors(sample, 'vs'),
vs_dark: extractColors(sample, 'vs-dark'),
hc_black: extractColors(sample, 'hc-black'),
}
};
TEST_CASES.push(testCase);
}
function extractColors(sample, theme) {
var out = document.createElement('pre');
var txt = document.createTextNode(sample.text);
out.appendChild(txt);
document.body.appendChild(out);
monaco.editor.colorizeElement(out, {
theme: theme,
mimeType: sample.language
});
if (GENERATE_TOKENS) {
var tokens = monaco.editor.tokenize(sample.text || '', sample.language);
}
// console.log(out);
var lineElement = out.firstElementChild;
var allResult = [];
while (lineElement) {
// console.log(lineElement);
var pieces = lineElement.children;
var result = [];
if (GENERATE_TOKENS) {
var lineTokens = tokens.shift();
if (lineTokens.length > 0) {
for (var i = 0; i < lineTokens.length - 1; i++) {
lineTokens[i].length = lineTokens[i + 1].offset - lineTokens[i].offset;
}
lineTokens[lineTokens.length - 1].length = 1000000;
}
}
for (var i = 0; i < pieces.length; i++) {
var piece = pieces[i];
// console.log(piece);
var text = piece.innerText;
var r = window.getComputedStyle(piece);
var color = rgbToHex(r.color);
// var isWhitespace = /^\s+$/.test(text);
// var partTokens = undefined;
if (GENERATE_TOKENS) {
let partTokens = [];
let partText = text;
while (lineTokens.length > 0 && partText.length > 0) {
let t = lineTokens.shift();
partTokens.push(t);
partText = partText.substr(t.length);
}
result.push({
text: text,
color: color,
tokens: partTokens
});
} else {
result.push({
text: text,
color: color,
});
}
}
allResult.push(result);
lineElement = lineElement.nextElementSibling.nextElementSibling;
}
// console.log(JSON.stringify(allResult));
out.parentNode.removeChild(out);
return allResult;
}
var rgbCache = {};
function rgbToHex(rgb) {
if (!rgbCache[rgb]) {
var m = rgb.match(/rgb\((\d+), (\d+), (\d+)\)/);
var r = parseInt(m[1], 10);
var g = parseInt(m[2], 10);
var b = parseInt(m[3], 10);
var r = '#' + to2Hex(r) + to2Hex(g) + to2Hex(b);
rgbCache[rgb] = r;
}
return rgbCache[rgb];
}
function to2Hex(n) {
var r = n.toString(16);
if (r.length === 1) {
return '0' + r;
}
return r;
}
function generateTests() {
// var ta = document.createElement('textarea');
// ta.value = JSON.stringify(TEST_CASES, null, '\t');
// ta.style.display = 'fixed';
// ta.style.top = '0';
// ta.style.left = '0';
// document.body.appendChild(ta);
// ta.select();
// return;
for (var i = 0, len = TEST_CASES.length; i < len; i++) {
if (LIMIT_TO_TEST && TEST_CASES[i].name !== LIMIT_TO_TEST) {
// if (TEST_CASES[i].name !== 'sample - html') {
continue;
}
generateTest(TEST_CASES[i], EXPECTED[i]);
}
QUnit.start();
}
function mergeEqualColors(source) {
// return source;
let result = [];
for (let i = 0; i < source.length; i++) {
let color = source[i].color;
let text = source[i].text;
if (result.length > 0 && result[result.length - 1].color === color) {
result[result.length - 1].text += text;
} else {
result.push({
color: color,
text: text
});
}
}
return result;
}
function generateTest(actual, expected) {
['vs', 'vs_dark', 'hc_black'].forEach(function(theme) {
// if (actual.name !== 'sample - bat') {
// return;
// }
QUnit.test(actual.name + ' ' + theme, function(assert) {
var _actual = actual.result[theme];
var _expected = expected.result[theme];
for (var i = 0; i < _actual.length; i++) {
var lineActual = mergeEqualColors(_actual[i]);
// .slice(0);
// for (var j = 1; j < lineActual.length; j++) {
// if (lineActual[j-1].color === lineActual[j].color) {
// lineActual[j-1] = {
// color: lineActual[j-1].color,
// text: lineActual[j-1].text + lineActual[j].text
// };
// lineActual.splice(j, 1);
// j--;
// }
// }
var lineExpected = mergeEqualColors(_expected[i]);
// .slice(0);
// for (var j = 1; j < lineExpected.length; j++) {
// if (lineExpected[j-1].color === lineExpected[j].color) {
// lineExpected[j-1] = {
// color: lineExpected[j-1].color,
// text: lineExpected[j-1].text + lineExpected[j].text
// };
// // lineExpected[j-1].text += lineExpected[j].text;
// lineExpected.splice(j, 1);
// j--;
// }
// }
assert.deepEqual(lineActual, lineExpected, 'Line #' + (i+1));
}
});
});
}
QUnit.config.autostart = false;
</script>
<div id="qunit"></div>
<div id="qunit-fixture"></div>
</body>
</html>