node.js for the rest of us

Simple things should be simple. Complex things should be possible.
Alan Kay

I published streamline.js 18 months ago but did not write a tutorial. I just took the time to do it.

The tutorial implements a simple search aggregator application. Here is a short spec for this application:

  • One page with a search field and a submit button.
  • The search is forwarded to Google and the results are displayed in the page.
  • A second search is run on the local tree of files. Matching files and lines are displayed.
  • A third search is run against a collection of movies in a MongoDB database. Matching movie titles and director names are displayed.
  • The 3 search operations are performed in parallel.
  • The file search is parallelized but limited to 100 simultaneous open files, to avoid running out of file descriptors on large trees.
  • The movies collection in MongoDB is automatically initialized with 4 entries the first time the application is run.

The implementation takes 126 lines (looks nicer in GitHub):

"use strict";
var streams = require('streamline/lib/streams/server/streams');
var url = require('url');
var qs = require('querystring');

var begPage = '<html><head><title>My Search</title></head></body>' + //
'<form action="/">Search: ' + //
'<input name="q" value="{q}"/>' + //
'<input type="submit"/>' + //
var endPage = '<hr/>generated in {ms}ms</body></html>';

streams.createHttpServer(function(request, response, _) {
  var query = qs.parse(url.parse(request.url).query),
    t0 = new Date();
  response.writeHead(200, {
    'Content-Type': 'text/html; charset=utf8'
  response.write(_, begPage.replace('{q}', query.q || ''));
  response.write(_, search(_, query.q));
  response.write(_, endPage.replace('{ms}', new Date() - t0));
}).listen(_, 1337);
console.log('Server running at');

function search(_, q) {
  if (!q || /^\s*$/.test(q)) return "Please enter a text to search";
  try {
    // start the 3 futures
    var googleFuture = googleSearch(null, q);
    var fileFuture = fileSearch(null, q);
    var mongoFuture = mongoSearch(null, q);
    // join the results
    return '<h2>Web</h2>' + googleFuture(_) //
    + '<hr/><h2>Files</h2>' + fileFuture(_) //
    + '<hr/><h2>Mongo</h2>' + mongoFuture(_);
  } catch (ex) {
    return 'an error occured. Retry or contact the site admin: ' + ex.stack;

function googleSearch(_, q) {
  var t0 = new Date();
  var json = streams.httpRequest({
    url: '' + q,
    proxy: process.env.http_proxy
  // parse JSON response
  var parsed = JSON.parse(json);
  // Google may refuse our request. Return the message then.
  if (!parsed.responseData) return "GOOGLE ERROR: " + parsed.responseDetails;
  // format result in HTML
  return '<ul>' + {
    return '<li><a href="' + entry.url + '">' + entry.titleNoFormatting + '</a></li>';
  }).join('') + '</ul>' + '<br/>completed in ' + (new Date() - t0) + ' ms';

var fs = require('fs'),
  flows = require('streamline/lib/util/flows');
// allocate a funnel for 100 concurrent open files
var filesFunnel = flows.funnel(100);

function fileSearch(_, q) {
  var t0 = new Date();
  var results = '';

  function doDir(_, dir) {
    fs.readdir(dir, _).forEach_(_, -1, function(_, file) {
      var f = dir + '/' + file;
      var stat = fs.stat(f, _);
      if (stat.isFile()) {
        // use the funnel to limit the number of open files 
        filesFunnel(_, function(_) {
          fs.readFile(f, 'utf8', _).split('\n').forEach(function(line, i) {
            if (line.indexOf(q) >= 0) results += '<br/>' + f + ':' + i + ':' + line;
      } else if (stat.isDirectory()) {
        doDir(_, f);
  doDir(_, __dirname);
  return results + '<br/>completed in ' + (new Date() - t0) + ' ms';;

var mongodb = require('mongodb'),
  mongoFunnel = flows.funnel(1);

function mongoSearch(_, q) {
  var t0 = new Date();
  var db = new mongodb.Db('tutorial', new mongodb.Server("", 27017, {}));;
  try {
    var coln = db.collection('movies', _);
    mongoFunnel(_, function(_) {
      if (coln.count(_) === 0) coln.insert(MOVIES, _);
    var re = new RegExp(".*" + q + ".*");
    return coln.find({
      $or: [{
        title: re
      }, {
        director: re
    }, _).toArray(_).map(function(movie) {
      return movie.title + ': ' + movie.director;
    }).join('<br/>') + '<br/>completed in ' + (new Date() - t0) + ' ms';;
  } finally {

var MOVIES = [{
  title: 'To be or not to be',
  director: 'Ernst Lubitsch'
}, {
  title: 'La Strada',
  director: 'Federico Fellini'
}, {
  title: 'Metropolis',
  director: 'Fritz Lang'
}, {
  title: 'Barry Lyndon',
  director: 'Stanley Kubrick'

I organized the tutorial in 7 steps but I did not have much to say at each step because it just all felt like normal JavaScript code around cool APIs, with the little _ to mark the spots where execution yields.

I’m blogging about it because I think that there is a real opportunity for node.js to attract mainstream programmers. And I feel that this is the kind of code that mainstream programmers would feel comfortable with.

This entry was posted in Asynchronous JavaScript, Uncategorized. Bookmark the permalink.

3 Responses to node.js for the rest of us

  1. spion says:

    Cool tutorial. My suggestion: add instructions on how would one use streamline with an expressjs-based http server. expressjs is unbelievably popular and adding it (at least as an alternative) in the tutorial will reassure many people that streamline.js is compatible with the existing tools and libraries that they use. Otherwise people might think that they must use streamline (which is not true)

    • Yes, it could be misleading. I did not want to have too many dependencies in the tutorial. But expressjs is very popular and I should add a step to demo how the two fit together.

      Will have to wait a bit though cause I’ll be without computer for two weeks 🙂

  2. Pingback: Node.Js: Links, News And Resources (16) | Angel "Java" Lopez on Blog

Leave a Reply to Bruno Jouhier Cancel reply

Fill in your details below or click an icon to log in: Logo

You are commenting using your account. Log Out /  Change )

Google photo

You are commenting using your Google account. Log Out /  Change )

Twitter picture

You are commenting using your Twitter account. Log Out /  Change )

Facebook photo

You are commenting using your Facebook account. Log Out /  Change )

Connecting to %s