Web Archiving and Analytics by Qumram: Lessons Learnt

Leonid Kyrpychenko

Qumram

@distributedLeo and @qumramAG

BarcelonaMay 19th 2015

2

3

What are the lessons learnt so far with Q5?

4

5

Lesson 1. Mocking is a MUST

stateManager = {frameSent: function () {}

};network = proxyquire('./network', {

'./stateManager': stateManager});

With AngularJS we use standard ngMock.

We use jasmine in front and backend (but not only).

For the browser applications we use Proxyquireify (requires browserify)

6

Lesson 2. DOM changes not always update innerHtml

// Risky Tag Names that should be handled manuallyvar tags = ['input', 'textarea', 'select', 'option'], // Risky Attributes to backup attributes = ['value', 'selected', 'checked', 'disabled'], // Risky Attributes that has no value noValueAttributes = ['selected', 'checked', 'disabled'];

7

Lesson 3. Compression and diffing without web workers

/** * Queue a work to the end of the javascript runtime excecution queue */ queueWork: function(context, fnNameOrReference, params, callback){ // add context to top of params, so that apply passes params correctly to bind params.unshift(context); // bind callback to context and add callback to end of parameter list params.push(callback.bind(context)); // we use apply on bind because the params in array, bind needs each params as argument, wheres // apply can take a paramter array, so we do apply on bind. if (typeof fnNameOrReference == 'string') { setTimeout(Function.bind.apply(context[fnNameOrReference],params),0); } else { setTimeout(Function.bind.apply(fnNameOrReference,params),0); } },

8

Lesson 4. Track events with different speed

To throttle the rate a function gets executed we use debouncing

var interval = config.get('recordMouseMovement').duration;

setInterval((function (buffer, activeMouseCordinates, lastRecordedMouseCordinates) { return recordMousePosition; })(buffer, activeMouseCordinates, lastRecordedMouseCordinates), interval)

self.api.deBouncedFilter = debounce(self.api.filter, 400);

9

Lesson 5. Parse html in a new iFrame was a huge bottleneck

var domParser = new DOMParser();domParser.parseFromString(html, 'text/html');

10

Lesson 6. Browserify

return gulp.src(['./components/tracker/index.js']) .pipe(transform(function(filename) { return browserify({ entries: filename, debug: true }).bundle(); })) .pipe(transform(function() { return exorcist('./build/qumram.tracker.js.map'); })) .pipe(rename('qumram.tracker.js'))…

11

Lesson 7. Promises and bluebird'use strict';

var bluebird = require('bluebird');var mongodb = require('mongodb');var config = require('./config.js');

bluebird.promisifyAll(mongodb);bluebird.promisifyAll(mongodb.MongoClient);bluebird.promisifyAll(mongodb.MongoClient.prototype);bluebird.promisifyAll(mongodb.Collection);bluebird.promisifyAll(mongodb.Collection.prototype);bluebird.promisifyAll(mongodb.Cursor.prototype);

module.exports = { /** * initialise database connection */ init: function () { return mongodb.MongoClient.connectAsync(config.dburl) .then(function (db) { module.exports.client = db; }); }};

12

Lesson 8. Koavar koa = require('koa');var cors = require('koa-cors');var socket = require('koa-socket');var qmrmRouter = require('qmrm-router');var http = require('http');

var app = koa();

app.use(cors());app.use(qmrmRouter.middleware());

socket.use(function *(next) { this.method = this.event.split(' ')[0].toUpperCase(); this.path = this.event.split(' ')[1]; yield next; this.socket.emit('resp ' + this.event, this.body);});

socket.use(qmrmRouter.middleware());

socket.start(app);

app.server = [];app.server[0] = http.createServer(app.callback());app.init();

app.server[0].listen(3000);

13

Lesson 9. Sockets + REST router = QRouter

npm install qrouter

var QmrmRouter = require('qrouter');

var router = new QmrmRouter();

router.options('/tokens', cors());router.post('post_tokens', 'ack.post_tokens', '/tokens', cors(), function (req, res) { var token = req.body;

github.com/qumram/qrouter

14

Join US!

Jobs@Qumram.com

15