NodeJS has a very usefully HTTP client library called request. It like an alternate of cURL in PHP. In this blog post we will see how to use this library and different case scenarios
The library is available at https://github.com/request/request and can be easily installed by “npm install request”.
Let start looking in various cases where this library can be used
Parsing a URL using NodeJS Request
In this example we will simple see how to parse a url or getting its source though the request library.
var request = require('request');
var zlib = require('zlib');
function getHTML(url, callback) {
var headers = {};
headers['user-agent'] = 'Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_0 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8A293 Safari/6531.22.7';
var options = {
url: url,
headers: headers
};
var req = request.get(options);
req.on('response', function (res) {
var chunks = [];
res.on('data', function (chunk) {
chunks.push(chunk);
});
res.on('end', function () {
var buffer = Buffer.concat(chunks);
var encoding = res.headers['content-encoding'];
if (encoding == 'gzip') {
zlib.gunzip(buffer, function (err, decoded) {
callback(err, decoded && decoded.toString());
});
} else if (encoding == 'deflate') {
zlib.inflate(buffer, function (err, decoded) {
callback(err, decoded && decoded.toString());
})
} else {
callback(null, buffer.toString());
}
});
});
req.on('error', function (err) {
callback(err);
});
}
getHTML('http://flipkart.com', function (err, html) {
if (err) {
console.log(err);
process.exit();
}
console.log(html);
});
In the above code, I have use zlib library in case the html recieved is compressed using gzip or deflate as in many cases. This is most simple use case of request library to parse a url.
POST or Form Submit Using Request
In the below code will see how to post a form using nodejs request library
var request = require('request');
var zlib = require('zlib');
function postForm(url, param, callback) {
var headers = {};
headers['user-agent'] = 'Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_0 like Mac OS X; en-us) AppleWebKit/532.9 (KHTML, like Gecko) Version/4.0.5 Mobile/8A293 Safari/6531.22.7';
var options = {
url: url,
headers: headers
};
headers['X-Requested-With'] = 'XMLHttpRequest'; //if ajax request
//headers['Referer'] = ”; //if you need to set referer header
headers['Content-Type'] = 'application/x-www-form-urlencoded; charset=UTF-8';
var j = request.jar()
var options = {
url: url,
headers: this.headers,
body: param,
json: true,
jar : j //for cookies
};
var req = request.post(options);
req.on('response', function (res) {
var chunks = [];
res.on('data', function (chunk) {
chunks.push(chunk);
});
res.on('end', function () {
var buffer = Buffer.concat(chunks);
var encoding = res.headers['content-encoding'];
if (encoding == 'gzip') {
zlib.gunzip(buffer, function (err, decoded) {
callback(err, decoded && decoded.toString());
});
} else if (encoding == 'deflate') {
zlib.inflate(buffer, function (err, decoded) {
callback(err, decoded && decoded.toString());
})
} else {
callback(null, buffer.toString());
}
});
});
req.on('error', function (err) {
callback(err);
});
}
postForm('https://www.flipkart.com/account/loginIframe', {
'email': '[email protected]',
'password': 'test1234',
'password_hash' : '======',
}, function (err, html) {
if (err) {
console.log(err);
process.exit();
}
console.log(html);
});
Send Request Payload
Many websites these days use Request Payload technique with their ajax request. Here is how it looks in network tab of chrome dev tools.
To automate such a request, here is the code.
var request = require('request');
var zlib = require('zlib');
function postForm(url, param, callback) {
var headers = {};
var options = {
url: url,
headers: headers
};
headers['X-Requested-With'] = 'XMLHttpRequest'; //if ajax request
headers['Content-Type'] = 'application/json; charset=utf-8';
var options = {
url: url,
headers: headers,
body: JSON.stringify(param)
};
var req = request.post(options);
req.on('response', function (res) {
var chunks = [];
res.on('data', function (chunk) {
chunks.push(chunk);
});
res.on('end', function () {
var buffer = Buffer.concat(chunks);
var encoding = res.headers['content-encoding'];
if (encoding == 'gzip') {
zlib.gunzip(buffer, function (err, decoded) {
callback(err, decoded && decoded.toString());
});
} else if (encoding == 'deflate') {
zlib.inflate(buffer, function (err, decoded) {
callback(err, decoded && decoded.toString());
})
} else {
callback(null, buffer.toString());
}
});
});
req.on('error', function (err) {
callback(err);
});
}
Basic HTTP Authorization with Request
Here is the code automated http authorization
var request = require('request');
var zlib = require('zlib');
function postForm(url, param, callback) {
var headers = {};
headers['Cache-Control'] = 'no-cache';
headers['Authorization'] = 'Basic ' + new Buffer('user:password').toString('base64')
var options = {
url: url,
headers: headers
};
var req = request.get(options);
req.on('response', function (res) {
var chunks = [];
res.on('data', function (chunk) {
chunks.push(chunk);
});
res.on('end', function () {
var buffer = Buffer.concat(chunks);
var encoding = res.headers['content-encoding'];
if (encoding == 'gzip') {
zlib.gunzip(buffer, function (err, decoded) {
callback(err, decoded && decoded.toString());
});
} else if (encoding == 'deflate') {
zlib.inflate(buffer, function (err, decoded) {
callback(err, decoded && decoded.toString());
})
} else {
callback(null, buffer.toString());
}
});
});
req.on('error', function (err) {
callback(err);
});
}
