2015-06-08 00:15:53 +00:00
|
|
|
/* eslint-disable no-process-exit */
|
|
|
|
require('dotenv').load();
|
2015-06-12 22:27:51 +00:00
|
|
|
var Rx = require('rx'),
|
2015-06-16 00:53:43 +00:00
|
|
|
uuid = require('node-uuid'),
|
|
|
|
assign = require('lodash/object/assign'),
|
|
|
|
mongodb = require('mongodb'),
|
|
|
|
secrets = require('../config/secrets');
|
2015-06-08 00:15:53 +00:00
|
|
|
|
|
|
|
var MongoClient = mongodb.MongoClient;
|
|
|
|
|
2015-06-12 22:27:51 +00:00
|
|
|
var providers = [
|
|
|
|
'facebook',
|
|
|
|
'twitter',
|
|
|
|
'google',
|
|
|
|
'github',
|
|
|
|
'linkedin'
|
|
|
|
];
|
|
|
|
|
2015-06-15 23:47:50 +00:00
|
|
|
// create async console.logs
|
|
|
|
function debug() {
|
|
|
|
var args = [].slice.call(arguments);
|
|
|
|
process.nextTick(function() {
|
|
|
|
console.log.apply(console, args);
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2015-06-08 00:15:53 +00:00
|
|
|
function createConnection(URI) {
|
|
|
|
return Rx.Observable.create(function(observer) {
|
|
|
|
MongoClient.connect(URI, function(err, database) {
|
|
|
|
if (err) {
|
|
|
|
return observer.onError(err);
|
|
|
|
}
|
|
|
|
observer.onNext(database);
|
2015-06-16 00:53:43 +00:00
|
|
|
observer.onCompleted();
|
2015-06-08 00:15:53 +00:00
|
|
|
});
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2015-06-08 05:06:57 +00:00
|
|
|
function createQuery(db, collection, options, batchSize) {
|
2015-06-08 00:15:53 +00:00
|
|
|
return Rx.Observable.create(function (observer) {
|
2015-06-08 05:06:57 +00:00
|
|
|
var cursor = db.collection(collection).find({}, options);
|
2015-06-08 00:15:53 +00:00
|
|
|
cursor.batchSize(batchSize || 20);
|
|
|
|
// Cursor.each will yield all doc from a batch in the same tick,
|
|
|
|
// or schedule getting next batch on nextTick
|
2015-06-15 23:47:50 +00:00
|
|
|
debug('opening cursor for %s', collection);
|
2015-06-08 00:15:53 +00:00
|
|
|
cursor.each(function (err, doc) {
|
|
|
|
if (err) {
|
|
|
|
return observer.onError(err);
|
|
|
|
}
|
|
|
|
if (!doc) {
|
2015-06-16 00:53:43 +00:00
|
|
|
console.log('onCompleted');
|
2015-06-08 00:15:53 +00:00
|
|
|
return observer.onCompleted();
|
|
|
|
}
|
|
|
|
observer.onNext(doc);
|
|
|
|
});
|
|
|
|
|
|
|
|
return Rx.Disposable.create(function () {
|
2015-06-15 23:47:50 +00:00
|
|
|
debug('closing cursor for %s', collection);
|
2015-06-08 00:15:53 +00:00
|
|
|
cursor.close();
|
|
|
|
});
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
2015-06-08 05:06:57 +00:00
|
|
|
function insertMany(db, collection, users, options) {
|
2015-06-08 00:15:53 +00:00
|
|
|
return Rx.Observable.create(function(observer) {
|
2015-06-08 05:06:57 +00:00
|
|
|
db.collection(collection).insertMany(users, options, function(err) {
|
2015-06-08 00:15:53 +00:00
|
|
|
if (err) {
|
|
|
|
return observer.onError(err);
|
|
|
|
}
|
2015-06-12 22:27:51 +00:00
|
|
|
observer.onNext();
|
2015-06-08 00:15:53 +00:00
|
|
|
observer.onCompleted();
|
|
|
|
});
|
|
|
|
});
|
|
|
|
}
|
|
|
|
|
|
|
|
var count = 0;
|
2015-06-08 05:06:57 +00:00
|
|
|
// will supply our db object
|
|
|
|
var dbObservable = createConnection(secrets.db).shareReplay();
|
2015-06-12 22:27:51 +00:00
|
|
|
|
|
|
|
var users = dbObservable
|
2015-06-08 00:15:53 +00:00
|
|
|
.flatMap(function(db) {
|
2015-06-08 05:06:57 +00:00
|
|
|
// returns user document, n users per loop where n is the batchsize.
|
2015-06-08 00:15:53 +00:00
|
|
|
return createQuery(db, 'users', {});
|
|
|
|
})
|
|
|
|
.map(function(user) {
|
2015-06-08 05:06:57 +00:00
|
|
|
// flatten user
|
2015-06-08 00:15:53 +00:00
|
|
|
assign(user, user.portfolio, user.profile);
|
|
|
|
return user;
|
|
|
|
})
|
2015-06-12 22:27:51 +00:00
|
|
|
.map(function(user) {
|
|
|
|
if (user.username) {
|
|
|
|
return user;
|
|
|
|
}
|
|
|
|
user.username = 'fcc' + uuid.v4().slice(0, 8);
|
|
|
|
return user;
|
|
|
|
})
|
|
|
|
.shareReplay();
|
|
|
|
|
|
|
|
// batch them into arrays of twenty documents
|
|
|
|
var userSavesCount = users
|
2015-06-08 05:06:57 +00:00
|
|
|
.bufferWithCount(20)
|
|
|
|
// get bd object ready for insert
|
|
|
|
.withLatestFrom(dbObservable, function(users, db) {
|
|
|
|
return {
|
|
|
|
users: users,
|
|
|
|
db: db
|
|
|
|
};
|
2015-06-08 00:15:53 +00:00
|
|
|
})
|
2015-06-08 05:06:57 +00:00
|
|
|
.flatMap(function(dats) {
|
|
|
|
// bulk insert into new collection for loopback
|
|
|
|
return insertMany(dats.db, 'user', dats.users, { w: 1 });
|
|
|
|
})
|
|
|
|
// count how many times insert completes
|
2015-06-12 22:27:51 +00:00
|
|
|
.count();
|
|
|
|
|
|
|
|
// create User Identities
|
|
|
|
var userIdentityCount = users
|
|
|
|
.flatMap(function(user) {
|
|
|
|
var ids = providers
|
|
|
|
.map(function(provider) {
|
|
|
|
return {
|
|
|
|
provider: provider,
|
2015-06-12 23:16:10 +00:00
|
|
|
externalId: user[provider],
|
|
|
|
userId: user.id
|
2015-06-12 22:27:51 +00:00
|
|
|
};
|
|
|
|
})
|
|
|
|
.filter(function(ident) {
|
|
|
|
return !!ident.externalId;
|
|
|
|
});
|
|
|
|
|
|
|
|
return Rx.Observable.from(ids);
|
|
|
|
})
|
|
|
|
.bufferWithCount(20)
|
|
|
|
.withLatestFrom(dbObservable, function(identities, db) {
|
|
|
|
return {
|
|
|
|
identities: identities,
|
|
|
|
db: db
|
|
|
|
};
|
|
|
|
})
|
|
|
|
.flatMap(function(dats) {
|
|
|
|
// bulk insert into new collection for loopback
|
|
|
|
return insertMany(dats.db, 'userIdentity', dats.identities, { w: 1 });
|
|
|
|
})
|
|
|
|
// count how many times insert completes
|
|
|
|
.count();
|
|
|
|
|
2015-06-15 23:47:50 +00:00
|
|
|
var storyCount = dbObservable
|
|
|
|
.flatMap(function(db) {
|
|
|
|
return createQuery(db, 'stories', {});
|
|
|
|
})
|
|
|
|
.bufferWithCount(20)
|
|
|
|
.withLatestFrom(dbObservable, function(stories, db) {
|
|
|
|
return {
|
|
|
|
stories: stories,
|
|
|
|
db: db
|
|
|
|
};
|
|
|
|
})
|
|
|
|
.flatMap(function(dats) {
|
2015-06-16 00:53:43 +00:00
|
|
|
return insertMany(dats.db, 'story', dats.stories, { w: 1 });
|
2015-06-15 23:47:50 +00:00
|
|
|
})
|
|
|
|
.count();
|
|
|
|
|
2015-06-26 04:47:25 +00:00
|
|
|
var commentCount = dbObservable
|
|
|
|
.flatMap(function(db) {
|
|
|
|
return createQuery(db, 'comments', {});
|
|
|
|
})
|
|
|
|
.withLatestFrom(dbObservable, function(comments, db) {
|
|
|
|
return {
|
|
|
|
comments: comments,
|
|
|
|
db: db
|
|
|
|
};
|
|
|
|
})
|
|
|
|
.flatMap(function(dats) {
|
|
|
|
return insertMany(dats.db, 'comment', dats.comments, { w: 1 });
|
|
|
|
})
|
|
|
|
.buffer(20)
|
|
|
|
.count();
|
|
|
|
|
2015-06-15 23:47:50 +00:00
|
|
|
Rx.Observable.combineLatest(
|
2015-06-12 22:27:51 +00:00
|
|
|
userIdentityCount,
|
2015-06-15 23:47:50 +00:00
|
|
|
userSavesCount,
|
|
|
|
storyCount,
|
2015-06-26 04:47:25 +00:00
|
|
|
commentCount,
|
|
|
|
function(userIdentCount, userCount, storyCount, commentCount) {
|
2015-06-15 23:47:50 +00:00
|
|
|
return {
|
|
|
|
userIdentCount: userIdentCount * 20,
|
|
|
|
userCount: userCount * 20,
|
2015-06-26 04:47:25 +00:00
|
|
|
storyCount: storyCount * 20,
|
|
|
|
commentCount: commentCount * 20
|
2015-06-15 23:47:50 +00:00
|
|
|
};
|
2015-06-16 00:53:43 +00:00
|
|
|
})
|
2015-06-08 00:15:53 +00:00
|
|
|
.subscribe(
|
2015-06-26 04:47:25 +00:00
|
|
|
function(countObj) {
|
|
|
|
console.log('next');
|
|
|
|
count = countObj;
|
|
|
|
},
|
|
|
|
function(err) {
|
|
|
|
console.error('an error occured', err, err.stack);
|
|
|
|
},
|
|
|
|
function() {
|
|
|
|
|
|
|
|
console.log('finished with ', count);
|
|
|
|
process.exit(0);
|
|
|
|
}
|
|
|
|
);
|