356 lines
10 KiB
JavaScript
356 lines
10 KiB
JavaScript
const path = require('path');
|
||
const liveServer = require('live-server');
|
||
|
||
const spinner = require('ora')();
|
||
|
||
const clientPath = path.resolve(__dirname, '../../client');
|
||
require('@babel/polyfill');
|
||
require('@babel/register')({
|
||
root: clientPath,
|
||
babelrc: false,
|
||
presets: ['@babel/preset-env'],
|
||
ignore: [/node_modules/],
|
||
only: [clientPath]
|
||
});
|
||
|
||
const createPseudoWorker = require('./utils/pseudo-worker');
|
||
const {
|
||
default: createWorker
|
||
} = require('../../client/src/templates/Challenges/utils/worker-executor');
|
||
|
||
const { assert, AssertionError } = require('chai');
|
||
const Mocha = require('mocha');
|
||
const { flatten } = require('lodash');
|
||
|
||
const jsdom = require('jsdom');
|
||
|
||
const dom = new jsdom.JSDOM('');
|
||
global.document = dom.window.document;
|
||
|
||
require('dotenv').config({ path: path.resolve(__dirname, '../../.env') });
|
||
|
||
const vm = require('vm');
|
||
|
||
const puppeteer = require('puppeteer');
|
||
|
||
const { getChallengesForLang } = require('../getChallenges');
|
||
|
||
const MongoIds = require('./utils/mongoIds');
|
||
const ChallengeTitles = require('./utils/challengeTitles');
|
||
const { challengeSchemaValidator } = require('../schema/challengeSchema');
|
||
const { challengeTypes } = require('../../client/utils/challengeTypes');
|
||
|
||
const { supportedLangs } = require('../utils');
|
||
|
||
const {
|
||
buildDOMChallenge,
|
||
buildJSChallenge
|
||
} = require('../../client/src/templates/Challenges/utils/build');
|
||
|
||
const {
|
||
createPoly
|
||
} = require('../../client/src/templates/Challenges/utils/polyvinyl');
|
||
|
||
const oldRunnerFail = Mocha.Runner.prototype.fail;
|
||
Mocha.Runner.prototype.fail = function(test, err) {
|
||
if (err instanceof AssertionError) {
|
||
const errMessage = String(err.message || '');
|
||
const assertIndex = errMessage.indexOf(': expected');
|
||
if (assertIndex !== -1) {
|
||
err.message = errMessage.slice(0, assertIndex);
|
||
}
|
||
// Don't show stacktrace for assertion errors.
|
||
if (err.stack) {
|
||
delete err.stack;
|
||
}
|
||
}
|
||
return oldRunnerFail.call(this, test, err);
|
||
};
|
||
|
||
async function newPageContext(browser) {
|
||
const page = await browser.newPage();
|
||
// it's needed for workers as context.
|
||
await page.goto('http://127.0.0.1:8080/index.html');
|
||
return page;
|
||
}
|
||
|
||
spinner.start();
|
||
spinner.text = 'Populate tests.';
|
||
|
||
let browser;
|
||
let page;
|
||
|
||
runTests();
|
||
|
||
async function runTests() {
|
||
let testLangs = [...supportedLangs];
|
||
if (process.env.TEST_CHALLENGES_FOR_LANGS) {
|
||
const filterLangs = process.env.TEST_CHALLENGES_FOR_LANGS.split(',').map(
|
||
lang => lang.trim().toLowerCase()
|
||
);
|
||
testLangs = testLangs.filter(lang => filterLangs.includes(lang));
|
||
}
|
||
|
||
const challenges = await Promise.all(
|
||
testLangs.map(lang => getChallenges(lang))
|
||
);
|
||
|
||
describe('Check challenges', function() {
|
||
before(async function() {
|
||
spinner.text = 'Testing';
|
||
this.timeout(50000);
|
||
liveServer.start({
|
||
host: '127.0.0.1',
|
||
port: '8080',
|
||
root: path.resolve(__dirname, 'stubs'),
|
||
mount: [['/js', path.join(clientPath, 'static/js')]],
|
||
open: false,
|
||
logLevel: 0
|
||
});
|
||
browser = await puppeteer.launch({
|
||
args: [
|
||
// Required for Docker version of Puppeteer
|
||
'--no-sandbox',
|
||
'--disable-setuid-sandbox',
|
||
// This will write shared memory files into /tmp instead of /dev/shm,
|
||
// because Docker’s default for /dev/shm is 64MB
|
||
'--disable-dev-shm-usage'
|
||
// dumpio: true
|
||
]
|
||
});
|
||
global.Worker = createPseudoWorker(await newPageContext(browser));
|
||
page = await newPageContext(browser);
|
||
await page.setViewport({ width: 300, height: 150 });
|
||
});
|
||
after(async function() {
|
||
this.timeout(30000);
|
||
if (browser) {
|
||
await browser.close();
|
||
}
|
||
liveServer.shutdown();
|
||
spinner.stop();
|
||
});
|
||
|
||
challenges.forEach(populateTestsForLang);
|
||
});
|
||
|
||
run();
|
||
}
|
||
|
||
async function getChallenges(lang) {
|
||
const challenges = await getChallengesForLang(lang).then(curriculum =>
|
||
Object.keys(curriculum)
|
||
.map(key => curriculum[key].blocks)
|
||
.reduce((challengeArray, superBlock) => {
|
||
const challengesForBlock = Object.keys(superBlock).map(
|
||
key => superBlock[key].challenges
|
||
);
|
||
return [...challengeArray, ...flatten(challengesForBlock)];
|
||
}, [])
|
||
);
|
||
return { lang, challenges };
|
||
}
|
||
|
||
function populateTestsForLang({ lang, challenges }) {
|
||
const mongoIds = new MongoIds();
|
||
const challengeTitles = new ChallengeTitles();
|
||
const validateChallenge = challengeSchemaValidator(lang);
|
||
|
||
describe(`Check challenges (${lang})`, function() {
|
||
this.timeout(5000);
|
||
|
||
challenges.forEach(challenge => {
|
||
describe(challenge.title || 'No title', function() {
|
||
it('Common checks', function() {
|
||
const result = validateChallenge(challenge);
|
||
if (result.error) {
|
||
throw new AssertionError(result.error);
|
||
}
|
||
const { id, title } = challenge;
|
||
mongoIds.check(id, title);
|
||
challengeTitles.check(title);
|
||
});
|
||
|
||
const { challengeType } = challenge;
|
||
if (
|
||
challengeType !== challengeTypes.html &&
|
||
challengeType !== challengeTypes.js &&
|
||
challengeType !== challengeTypes.bonfire &&
|
||
challengeType !== challengeTypes.modern &&
|
||
challengeType !== challengeTypes.backend
|
||
) {
|
||
return;
|
||
}
|
||
|
||
let { tests = [] } = challenge;
|
||
tests = tests.filter(test => !!test.testString);
|
||
if (tests.length === 0) {
|
||
it('Check tests. No tests.');
|
||
return;
|
||
}
|
||
|
||
describe('Check tests syntax', function() {
|
||
tests.forEach(test => {
|
||
it(`Check for: ${test.text}`, function() {
|
||
assert.doesNotThrow(() => new vm.Script(test.testString));
|
||
});
|
||
});
|
||
});
|
||
|
||
let { files = [] } = challenge;
|
||
let createTestRunner;
|
||
if (challengeType === challengeTypes.backend) {
|
||
it('Check tests is not implemented.');
|
||
return;
|
||
} else if (
|
||
challengeType === challengeTypes.js ||
|
||
challengeType === challengeTypes.bonfire
|
||
) {
|
||
createTestRunner = createTestRunnerForJSChallenge;
|
||
} else if (files.length === 1) {
|
||
createTestRunner = createTestRunnerForDOMChallenge;
|
||
} else {
|
||
it('Check tests.', () => {
|
||
throw new Error('Seed file should be only the one.');
|
||
});
|
||
return;
|
||
}
|
||
|
||
files = files.map(createPoly);
|
||
it('Test suite must fail on the initial contents', async function() {
|
||
this.timeout(5000 * tests.length + 1000);
|
||
// suppress errors in the console.
|
||
const oldConsoleError = console.error;
|
||
console.error = () => {};
|
||
let fails = false;
|
||
let testRunner;
|
||
try {
|
||
testRunner = await createTestRunner(
|
||
{ ...challenge, files },
|
||
'',
|
||
page
|
||
);
|
||
} catch {
|
||
fails = true;
|
||
}
|
||
if (!fails) {
|
||
for (const test of tests) {
|
||
try {
|
||
await testRunner(test);
|
||
} catch (e) {
|
||
fails = true;
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
console.error = oldConsoleError;
|
||
assert(fails, 'Test suit does not fail on the initial contents');
|
||
});
|
||
|
||
let { solutions = [] } = challenge;
|
||
const noSolution = new RegExp('// solution required');
|
||
solutions = solutions.filter(
|
||
solution => !!solution && !noSolution.test(solution)
|
||
);
|
||
|
||
if (solutions.length === 0) {
|
||
it('Check tests. No solutions');
|
||
return;
|
||
}
|
||
|
||
describe('Check tests against solutions', function() {
|
||
solutions.forEach((solution, index) => {
|
||
it(`Solution ${index + 1}`, async function() {
|
||
this.timeout(5000 * tests.length + 1000);
|
||
const testRunner = await createTestRunner(
|
||
{ ...challenge, files },
|
||
solution,
|
||
page
|
||
);
|
||
for (const test of tests) {
|
||
await testRunner(test);
|
||
}
|
||
});
|
||
});
|
||
});
|
||
});
|
||
});
|
||
});
|
||
}
|
||
|
||
async function createTestRunnerForDOMChallenge(
|
||
{ required = [], template, files },
|
||
solution,
|
||
context
|
||
) {
|
||
if (solution) {
|
||
files[0].contents = solution;
|
||
}
|
||
|
||
const { build, sources, loadEnzyme } = await buildDOMChallenge({
|
||
files,
|
||
required,
|
||
template
|
||
});
|
||
|
||
await context.reload();
|
||
await context.setContent(build);
|
||
await context.evaluate(
|
||
async (sources, loadEnzyme) => {
|
||
const code = sources && 'index' in sources ? sources['index'] : '';
|
||
const getUserInput = fileName => sources[fileName];
|
||
await document.__initTestFrame({ code, getUserInput, loadEnzyme });
|
||
},
|
||
sources,
|
||
loadEnzyme
|
||
);
|
||
|
||
return async ({ text, testString }) => {
|
||
try {
|
||
const { pass, err } = await Promise.race([
|
||
new Promise((_, reject) => setTimeout(() => reject('timeout'), 5000)),
|
||
await context.evaluate(async testString => {
|
||
return await document.__runTest(testString);
|
||
}, testString)
|
||
]);
|
||
if (!pass) {
|
||
throw AssertionError(`${text}\n${err.message}`);
|
||
}
|
||
} catch (err) {
|
||
throw typeof err === 'string'
|
||
? `${text}\n${err}`
|
||
: (err.message = `${text}
|
||
${err.message}`);
|
||
}
|
||
};
|
||
}
|
||
|
||
async function createTestRunnerForJSChallenge({ files }, solution) {
|
||
if (solution) {
|
||
files[0].contents = solution;
|
||
}
|
||
|
||
const { build, sources } = await buildJSChallenge({ files });
|
||
const code = sources && 'index' in sources ? sources['index'] : '';
|
||
|
||
const testWorker = createWorker('test-evaluator');
|
||
return async ({ text, testString }) => {
|
||
try {
|
||
const { pass, err } = await testWorker.execute(
|
||
{ testString, build, code, sources },
|
||
5000
|
||
);
|
||
if (!pass) {
|
||
throw new AssertionError(`${text}\n${err.message}`);
|
||
}
|
||
} catch (err) {
|
||
throw typeof err === 'string'
|
||
? `${text}\n${err}`
|
||
: (err.message = `${text}
|
||
${err.message}`);
|
||
} finally {
|
||
testWorker.killWorker();
|
||
}
|
||
};
|
||
}
|