const path = require('path'); const liveServer = require('live-server'); const spinner = require('ora')(); const clientPath = path.resolve(__dirname, '../../client'); require('@babel/polyfill'); require('@babel/register')({ root: clientPath, babelrc: false, presets: ['@babel/preset-env'], ignore: [/node_modules/], only: [clientPath] }); const createPseudoWorker = require('./utils/pseudo-worker'); const { default: createWorker } = require('../../client/src/templates/Challenges/utils/worker-executor'); const { assert, AssertionError } = require('chai'); const Mocha = require('mocha'); const { flatten } = require('lodash'); const jsdom = require('jsdom'); const dom = new jsdom.JSDOM(''); global.document = dom.window.document; const vm = require('vm'); const puppeteer = require('puppeteer'); const { getChallengesForLang } = require('../getChallenges'); const MongoIds = require('./utils/mongoIds'); const ChallengeTitles = require('./utils/challengeTitles'); const { challengeSchemaValidator } = require('../schema/challengeSchema'); const { challengeTypes } = require('../../client/utils/challengeTypes'); const { testedLangs } = require('../utils'); const { buildDOMChallenge, buildJSChallenge } = require('../../client/src/templates/Challenges/utils/build'); const { createPoly } = require('../../client/src/templates/Challenges/utils/polyvinyl'); const oldRunnerFail = Mocha.Runner.prototype.fail; Mocha.Runner.prototype.fail = function(test, err) { if (err instanceof AssertionError) { const errMessage = String(err.message || ''); const assertIndex = errMessage.indexOf(': expected'); if (assertIndex !== -1) { err.message = errMessage.slice(0, assertIndex); } // Don't show stacktrace for assertion errors. if (err.stack) { delete err.stack; } } return oldRunnerFail.call(this, test, err); }; async function newPageContext(browser) { const page = await browser.newPage(); // it's needed for workers as context. await page.goto('http://127.0.0.1:8080/index.html'); return page; } spinner.start(); spinner.text = 'Populate tests.'; let browser; let page; runTests(); async function runTests() { process.on('unhandledRejection', err => { spinner.stop(); throw new Error(`unhandledRejection: ${err.name}, ${err.message}`); }); const testLangs = testedLangs(); const challenges = await Promise.all( testLangs.map(lang => getChallenges(lang)) ); describe('Check challenges', function() { before(async function() { spinner.text = 'Testing'; this.timeout(50000); liveServer.start({ host: '127.0.0.1', port: '8080', root: path.resolve(__dirname, 'stubs'), mount: [['/js', path.join(clientPath, 'static/js')]], open: false, logLevel: 0 }); browser = await puppeteer.launch({ args: [ // Required for Docker version of Puppeteer '--no-sandbox', '--disable-setuid-sandbox', // This will write shared memory files into /tmp instead of /dev/shm, // because Docker’s default for /dev/shm is 64MB '--disable-dev-shm-usage' // dumpio: true ] }); global.Worker = createPseudoWorker(await newPageContext(browser)); page = await newPageContext(browser); await page.setViewport({ width: 300, height: 150 }); }); after(async function() { this.timeout(30000); if (browser) { await browser.close(); } liveServer.shutdown(); spinner.stop(); }); challenges.forEach(populateTestsForLang); }); run(); } async function getChallenges(lang) { const challenges = await getChallengesForLang(lang).then(curriculum => Object.keys(curriculum) .map(key => curriculum[key].blocks) .reduce((challengeArray, superBlock) => { const challengesForBlock = Object.keys(superBlock).map( key => superBlock[key].challenges ); return [...challengeArray, ...flatten(challengesForBlock)]; }, []) ); return { lang, challenges }; } function populateTestsForLang({ lang, challenges }) { const mongoIds = new MongoIds(); const challengeTitles = new ChallengeTitles(); const validateChallenge = challengeSchemaValidator(lang); describe(`Check challenges (${lang})`, function() { this.timeout(5000); challenges.forEach(challenge => { describe(challenge.title || 'No title', function() { it('Common checks', function() { const result = validateChallenge(challenge); if (result.error) { throw new AssertionError(result.error); } const { id, title } = challenge; mongoIds.check(id, title); challengeTitles.check(title); }); const { challengeType } = challenge; if ( challengeType !== challengeTypes.html && challengeType !== challengeTypes.js && challengeType !== challengeTypes.bonfire && challengeType !== challengeTypes.modern && challengeType !== challengeTypes.backend ) { return; } let { tests = [] } = challenge; tests = tests.filter(test => !!test.testString); if (tests.length === 0) { it('Check tests. No tests.'); return; } describe('Check tests syntax', function() { tests.forEach(test => { it(`Check for: ${test.text}`, function() { assert.doesNotThrow(() => new vm.Script(test.testString)); }); }); }); let { files = [] } = challenge; let createTestRunner; if (challengeType === challengeTypes.backend) { it('Check tests is not implemented.'); return; } else if ( challengeType === challengeTypes.js || challengeType === challengeTypes.bonfire ) { createTestRunner = createTestRunnerForJSChallenge; } else if (files.length === 1) { createTestRunner = createTestRunnerForDOMChallenge; } else { it('Check tests.', () => { throw new Error('Seed file should be only the one.'); }); return; } files = files.map(createPoly); it('Test suite must fail on the initial contents', async function() { this.timeout(5000 * tests.length + 1000); // suppress errors in the console. const oldConsoleError = console.error; console.error = () => {}; let fails = false; let testRunner; try { testRunner = await createTestRunner( { ...challenge, files }, '', page ); } catch { fails = true; } if (!fails) { for (const test of tests) { try { await testRunner(test); } catch (e) { fails = true; break; } } } console.error = oldConsoleError; assert(fails, 'Test suit does not fail on the initial contents'); }); let { solutions = [] } = challenge; const noSolution = new RegExp('// solution required'); solutions = solutions.filter( solution => !!solution && !noSolution.test(solution) ); if (solutions.length === 0) { it('Check tests. No solutions'); return; } describe('Check tests against solutions', function() { solutions.forEach((solution, index) => { it(`Solution ${index + 1} must pass the tests`, async function() { this.timeout(5000 * tests.length + 1000); const testRunner = await createTestRunner( { ...challenge, files }, solution, page ); for (const test of tests) { await testRunner(test); } }); }); }); }); }); }); } async function createTestRunnerForDOMChallenge( { required = [], template, files }, solution, context ) { if (solution) { files[0].contents = solution; } const { build, sources, loadEnzyme } = await buildDOMChallenge({ files, required, template }); await context.reload(); await context.setContent(build); await context.evaluate( async (sources, loadEnzyme) => { const code = sources && 'index' in sources ? sources['index'] : ''; const getUserInput = fileName => sources[fileName]; await document.__initTestFrame({ code, getUserInput, loadEnzyme }); }, sources, loadEnzyme ); return async ({ text, testString }) => { try { const { pass, err } = await Promise.race([ new Promise((_, reject) => setTimeout(() => reject('timeout'), 5000)), await context.evaluate(async testString => { return await document.__runTest(testString); }, testString) ]); if (!pass) { throw new AssertionError(err.message); } } catch (err) { reThrow(err, text); } }; } async function createTestRunnerForJSChallenge({ files }, solution) { if (solution) { files[0].contents = solution; } const { build, sources } = await buildJSChallenge({ files }); const code = sources && 'index' in sources ? sources['index'] : ''; const testWorker = createWorker('test-evaluator', { terminateWorker: true }); return async ({ text, testString }) => { try { const { pass, err } = await testWorker.execute( { testString, build, code, sources }, 5000 ).done; if (!pass) { throw new AssertionError(err.message); } } catch (err) { reThrow(err, text); } }; } function reThrow(err, text) { if (typeof err === 'string') { throw new AssertionError( `${text} ${err}` ); } else { err.message = `${text} ${err.message}`; throw err; } }