greatbody
/
freeCodeCamp
mirror of https://github.com/freeCodeCamp/freeCodeCamp.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

chore(seed): Move seed script to tools

pull/18270/head
Bouncey 2018-10-07 08:51:58 +01:00 committed by mrugesh mohapatra
parent 26750776ed
commit bc9b3b4ddd
13 changed files with 6433 additions and 1801 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
api-server/server/utils/index.js
View File

@ -1,4 +1,4 @@
export function dasherize(name) {
exports.dasherize = function dasherize(name) {
return ('' + name)
.toLowerCase()
.replace(/\s/g, '-')
@ -6,13 +6,13 @@ export function dasherize(name) {
.replace(/\:/g, '');
}
export function nameify(str) {
exports.nameify = function nameify(str) {
return ('' + str)
.replace(/[^a-zA-Z0-9\s]/g, '')
.replace(/\:/g, '');
}
export function unDasherize(name) {
exports.unDasherize = function unDasherize(name) {
return ('' + name)
// replace dash with space
.replace(/\-/g, ' ')
@ -21,6 +21,6 @@ export function unDasherize(name) {
.trim();
}
export function addPlaceholderImage(name) {
exports.addPlaceholderImage = function addPlaceholderImage(name) {
return `https://identicon.org?t=${name}&s=256`;
}

3
package-lock.json generated
View File

@ -2234,7 +2234,8 @@
"dotenv": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-6.0.0.tgz",
"integrity": "sha512-FlWbnhgjtwD+uNLUGHbMykMOYQaTivdHEmYwAKFjn6GKe/CqY0fNae93ZHTd20snh9ZLr8mTzIL9m0APQ1pjQg=="
"integrity": "sha512-FlWbnhgjtwD+uNLUGHbMykMOYQaTivdHEmYwAKFjn6GKe/CqY0fNae93ZHTd20snh9ZLr8mTzIL9m0APQ1pjQg==",
"dev": true
},
"duplexer": {
"version": "0.1.1",

7
package.json
View File

@ -7,20 +7,19 @@
"ensure-env": "node ./tools/scripts/ensure-env.js",
"lint": "echo 'Warning: TODO - Define Linting.'",
"pretest": "npm-run-all -s lint bootstrap",
"seed": "node seed/index.js",
"seed": "node seed/seedChallenges",
"test": "npm-run-all -p test:*",
"test:client": "cd ./client && npm test && cd ../",
"test:curriculum": "echo 'Warning: TODO - Define Testing.'",
"test:server": "echo 'Warning: TODO - Define Testing.'",
"test:tools": "cd ./tools/challenge-md-parser && npm test && cd ../scripts/seed && npm test && cd ../../../",
"start-develop": "node ./tools/scripts/start-develop.js"
},
"devDependencies": {
"dotenv": "^6.0.0",
"eslint-config-freecodecamp": "^1.1.1",
"lerna": "^3.4.0",
"npm-run-all": "^4.1.3",
"tree-kill": "^1.2.0"
},
"dependencies": {
"dotenv": "^6.0.0"
}
}

52
seed/addAssertsToTapTest.js
View File

@ -1,52 +0,0 @@
let _ = require('lodash');
function createIsAssert(tapTest, isThing) {
const { assert } = tapTest;
return function() {
const args = [...arguments];
args[0] = isThing(args[0]);
assert.apply(tapTest, args);
};
}
function addAssertsToTapTest(tapTest) {
const assert = tapTest.assert;
assert.isArray = createIsAssert(tapTest, _.isArray);
assert.isBoolean = createIsAssert(tapTest, _.isBoolean);
assert.isString = createIsAssert(tapTest, _.isString);
assert.isNumber = createIsAssert(tapTest, _.isNumber);
assert.isUndefined = createIsAssert(tapTest, _.isUndefined);
assert.deepEqual = tapTest.deepEqual;
assert.equal = tapTest.equal;
assert.strictEqual = tapTest.equal;
assert.sameMembers = function sameMembers() {
const [ first, second, ...args] = arguments;
assert.apply(
tapTest,
[
_.difference(first, second).length === 0 &&
_.difference(second, first).length === 0
].concat(args)
);
};
assert.includeMembers = function includeMembers() {
const [ first, second, ...args] = arguments;
assert.apply(tapTest,
[
_.difference(second, first).length === 0
].concat(args));
};
assert.match = function match() {
const [value, regex, ...args] = arguments;
assert.apply(tapTest,
[
regex.test(value)
].concat(args));
};
return assert;
}
module.exports = addAssertsToTapTest;

28
seed/challengeTitles.js
View File

@ -1,28 +0,0 @@
class ChallengeTitles {
constructor() {
this.knownTitles = [];
}
check(title) {
if (typeof title !== 'string') {
throw new Error(`
Expected a valid string for ${title}, but got a(n) ${typeof title}
`);
} else if (title.length === 0) {
throw new Error(`
Expected a title length greater than 0
`);
}
const titleToCheck = title.toLowerCase().replace(/\s+/g, '');
const isKnown = this.knownTitles.includes(titleToCheck);
if (isKnown) {
throw new Error(`
All challenges must have a unique title.
The title ${title} is already assigned
`);
}
this.knownTitles = [ ...this.knownTitles, titleToCheck ];
}
}
export default ChallengeTitles;

74
seed/createPathMigrationMap.js
View File

@ -1,74 +0,0 @@
require('babel-register');
const { getChallenges } = require('@freecodecamp/curriculum');
const fs = require('fs');
const path = require('path');
const { Observable } = require('rx');
const { dasherize } = require('../api-server/server/utils');
let pathMap = {};
function createPathMigrationMap() {
return new Promise(resolve => {
Observable.of(getChallenges())
.map(blocks => {
blocks.forEach(block => {
const { name: blockName, superBlock, challenges } = block;
if (!(dasherize(superBlock) in pathMap)) {
pathMap[dasherize(superBlock)] = {};
}
if (!(dasherize(blockName) in pathMap[superBlock])) {
pathMap[dasherize(superBlock)][
dasherize(blockName)
] = challenges.map(({ title, challengeType }) => ({
dashedName: dasherize(title),
challengeType
}));
}
});
})
.subscribe(() => {}, console.error, () => {
const migMap = Object.keys(pathMap)
.filter(key => !key.includes('certificate'))
.map(superBlock => {
return Object.keys(pathMap[superBlock])
.map(block => {
return pathMap[superBlock][block].reduce(
(map, { dashedName, challengeType }) => ({
...map,
[dashedName]:
challengeType === 7
? `/${superBlock}/${block}`
: `/${superBlock}/${block}/${dashedName}`
}),
{}
);
})
.reduce(
(acc, current) => ({
...acc,
...current
}),
{}
);
})
.reduce(
(acc, current) => ({
...acc,
...current
}),
{}
);
fs.writeFileSync(
path.resolve(
__dirname,
'../api-server/server/resources/pathMigration.json'
),
JSON.stringify(migMap, null, 2)
);
resolve();
});
});
}
exports.createPathMigrationMap = createPathMigrationMap;

24
seed/mongoIds.js
View File

@ -1,24 +0,0 @@
import _ from 'lodash';
import { isMongoId } from 'validator';
class MongoIds {
constructor() {
this.knownIds = [];
}
check(id, title) {
if (!isMongoId(id)) {
throw new Error(`Expected a valid ObjectId for ${title}, but got ${id}`);
}
const idIndex = _.findIndex(this.knownIds, existing => id === existing);
if (idIndex !== -1) {
throw new Error(`
All challenges must have a unique id.
The id for ${title} is already assigned
`);
}
this.knownIds = [ ...this.knownIds, id ];
}
}
export default MongoIds;

1290
seed/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

77
seed/schema/challengeSchema.js
View File

@ -1,77 +0,0 @@
const Joi = require('joi');
Joi.objectId = require('joi-objectid')(Joi);
const schema = Joi.object().keys({
block: Joi.string(),
blockId: Joi.objectId(),
challengeType: Joi.number().integer().min(0).max(9).required().options({ convert: false }),
checksum: Joi.number(),
dashedName: Joi.string(),
description: Joi.array().items(
Joi.string().allow('')
).required(),
fileName: Joi.string(),
files: Joi.object().pattern(
/(jsx?|html|css|sass)$/,
Joi.object().keys({
key: Joi.string(),
ext: Joi.string(),
name: Joi.string(),
head: [
Joi.array().items(Joi.string().allow('')),
Joi.string().allow('')
],
tail: [
Joi.array().items(Joi.string().allow('')),
Joi.string().allow('')
],
contents: [
Joi.array().items(Joi.string().allow('')),
Joi.string().allow('')
]
})
),
guideUrl: Joi.string().uri({ scheme: 'https' }),
helpRoom: Joi.string(),
id: Joi.objectId().required(),
isBeta: Joi.bool(),
isComingSoon: Joi.bool(),
isLocked: Joi.bool(),
isPrivate: Joi.bool(),
isRequired: Joi.bool(),
name: Joi.string(),
order: Joi.number().options({ convert: false }),
required: Joi.array().items(
Joi.object().keys({
link: Joi.string(),
raw: Joi.bool(),
src: Joi.string(),
crossDomain: Joi.bool()
})
),
solutions: Joi.array().items(
Joi.string().optional()
),
superBlock: Joi.string(),
superOrder: Joi.number().options({ convert: false }),
suborder: Joi.number().options({ convert: false }),
tests: Joi.array().items(
// public challenges
Joi.object().keys({
text: Joi.string().required(),
testString: Joi.string().allow('').required()
}),
// our tests used in certification verification
Joi.object().keys({
id: Joi.string().required(),
title: Joi.string().required()
})
),
template: Joi.string(),
time: Joi.string().allow(''),
title: Joi.string().required()
});
exports.validateChallenge = function validateChallenge(challenge) {
return Joi.validate(challenge, schema);
};

244
seed/test-challenges.js
View File

@ -1,244 +0,0 @@
/* eslint-disable no-eval, no-process-exit, no-unused-vars */
import {Observable} from 'rx';
import tape from 'tape';
import { getChallenges } from '@freecodecamp/curriculum';
import MongoIds from './mongoIds';
import ChallengeTitles from './challengeTitles';
import addAssertsToTapTest from './addAssertsToTapTest';
import { validateChallenge } from './schema/challengeSchema';
// modern challengeType
const modern = 6;
let mongoIds = new MongoIds();
let challengeTitles = new ChallengeTitles();
function evaluateTest(
solution,
assert,
react,
redux,
reactRedux,
head,
tail,
test,
tapTest
) {
let code = solution;
/* NOTE: Provide dependencies for React/Redux challenges
* and configure testing environment
*/
let React,
ReactDOM,
Redux,
ReduxThunk,
ReactRedux,
Enzyme,
document;
// Fake Deep Equal dependency
const DeepEqual = (a, b) =>
JSON.stringify(a) === JSON.stringify(b);
// Hardcode Deep Freeze dependency
const DeepFreeze = (o) => {
Object.freeze(o);
Object.getOwnPropertyNames(o).forEach(function(prop) {
if (o.hasOwnProperty(prop)
&& o[ prop ] !== null
&& (
typeof o[ prop ] === 'object' ||
typeof o[ prop ] === 'function'
)
&& !Object.isFrozen(o[ prop ])) {
DeepFreeze(o[ prop ]);
}
});
return o;
};
if (react || redux || reactRedux) {
// Provide dependencies, just provide all of them
React = require('react');
ReactDOM = require('react-dom');
Redux = require('redux');
ReduxThunk = require('redux-thunk');
ReactRedux = require('react-redux');
Enzyme = require('enzyme');
const Adapter15 = require('enzyme-adapter-react-15');
Enzyme.configure({ adapter: new Adapter15() });
/* Transpile ALL the code
* (we may use JSX in head or tail or tests, too): */
const transform = require('babel-standalone').transform;
const options = { presets: [ 'es2015', 'react' ] };
head = transform(head, options).code;
solution = transform(solution, options).code;
tail = transform(tail, options).code;
test = transform(test, options).code;
const { JSDOM } = require('jsdom');
// Mock DOM document for ReactDOM.render method
const jsdom = new JSDOM(`<!doctype html>
<html>
<body>
<div id="challenge-node"></div>
</body>
</html>
`);
const { window } = jsdom;
// Mock DOM for ReactDOM tests
document = window.document;
global.window = window;
global.document = window.document;
}
/* eslint-enable no-unused-vars */
try {
(() => {
return eval(
head + '\n' +
solution + '\n' +
tail + '\n' +
test.testString
);
})();
} catch (e) {
console.log(
head + '\n' +
solution + '\n' +
tail + '\n' +
test.testString
);
console.log(e);
tapTest.fail(e);
process.exit(1);
}
}
function createTest({
title,
id = '',
tests = [],
solutions = [],
files = [],
react = false,
redux = false,
reactRedux = false
}) {
mongoIds.check(id, title);
challengeTitles.check(title);
solutions = solutions.filter(solution => !!solution);
tests = tests.filter(test => !!test);
// No support for async tests
const isAsync = s => s.includes('(async () => ');
if (isAsync(tests.join(''))) {
console.log(`Replacing Async Tests for Challenge ${title}`);
tests = tests.map(challengeTestSource =>
isAsync(challengeTestSource) ?
"assert(true, 'message: great');" :
challengeTestSource);
}
const { head, tail } = Object.keys(files)
.map(key => files[key])
.reduce(
(result, file) => ({
head: result.head + ';' + file.head.join('\n'),
tail: result.tail + ';' + file.tail.join('\n')
}),
{ head: '', tail: '' }
);
const plan = tests.length;
if (!plan) {
return Observable.just({
title,
type: 'missing'
});
}
return Observable.fromCallback(tape)(title)
.doOnNext(tapTest =>
solutions.length ? tapTest.plan(plan) : tapTest.end())
.flatMap(tapTest => {
if (solutions.length <= 0) {
tapTest.comment('No solutions for ' + title);
return Observable.just({
title,
type: 'missing'
});
}
return Observable.just(tapTest)
.map(addAssertsToTapTest)
/* eslint-disable no-unused-vars */
// assert and code used within the eval
.doOnNext(assert => {
solutions.forEach(solution => {
tests.forEach(test => {
evaluateTest(
solution,
assert,
react,
redux,
reactRedux,
head,
tail,
test,
tapTest
);
});
});
})
.map(() => ({ title }));
});
}
Observable.from(getChallenges())
.do(({ challenges }) => {
challenges.forEach(challenge => {
const result = validateChallenge(challenge);
if (result.error) {
console.log(result.value);
throw new Error(result.error);
}
});
})
.flatMap(challengeSpec => {
return Observable.from(challengeSpec.challenges);
})
.filter(({ challengeType }) => challengeType !== modern)
.flatMap(challenge => {
return createTest(challenge);
})
.map(({ title, type }) => {
if (type === 'missing') {
return title;
}
return false;
})
.filter(title => !!title)
.toArray()
.subscribe(
(noSolutions) => {
if (noSolutions) {
console.log(
'# These challenges have no solutions\n- [ ] ' +
noSolutions.join('\n- [ ] ')
);
}
},
err => {
throw err;
},
() => process.exit(0)
);

6401
tools/scripts/seed/package-lock.json generated Normal file
View File

File diff suppressed because it is too large Load Diff

4
seed/package.json → tools/scripts/seed/package.json
View File

@ -4,7 +4,7 @@
"description": "A script to seed challenges in to the database",
"main": "index.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
"test": "jest"
},
"repository": {
"type": "git",
@ -18,7 +18,9 @@
"homepage": "https://github.com/freeCodeCamp/freeCodeCamp#readme",
"devDependencies": {
"@freecodecamp/curriculum": "0.0.0-next.4",
"debug": "^4.0.1",
"dotenv": "^6.0.0",
"jest": "^23.6.0",
"lodash": "^4.17.11",
"mongodb": "^3.1.6"
}

22
seed/index.js → tools/scripts/seed/seedChallenges.js
View File

@ -1,9 +1,14 @@
const path = require('path');
const fs = require('fs');
require('dotenv').config({ path: path.resolve(__dirname, '../.env') });
const MongoClient = require('mongodb').MongoClient;
const { getChallengesForLang } = require('@freecodecamp/curriculum');
const { flatten } = require('lodash');
const debug = require('debug');
const { createPathMigrationMap } = require('./createPathMigrationMap');
const log = debug('fcc:tools:seedChallenges');
const { MONGOHQ_URL, LOCALE: lang } = process.env;
function handleError(err, client) {
@ -27,7 +32,7 @@ MongoClient.connect(
function(err, client) {
handleError(err, client);
console.log('Connected successfully to mongo');
log('Connected successfully to mongo');
const db = client.db('freecodecamp');
const challenges = db.collection('challenges');
@ -51,8 +56,21 @@ MongoClient.connect(
} catch (e) {
handleError(e, client);
} finally {
console.log('challenge seed complete');
log('challenge seed complete');
client.close();
log('generating path migration map');
const pathMap = createPathMigrationMap(curriculum);
const outputDir = path.resolve(
__dirname,
'../../../api-server/server/resources/pathMigration.json'
);
fs.writeFile(outputDir, JSON.stringify(pathMap), err => {
if (err) {
console.error('Oh noes!!');
console.error(err);
}
log('path migration map generated');
});
}
});
}