I am trying to login into an online shopping site from Chrome extension, then scrape it. There are three steps that happens when you do this from a browser.
- Visit the site, the site gives you a cookie.
- Go to the login page, send the aforementioned cookie, and username and password as params in POST. The site gives you 5 more cookies.
- call GET to a path within the site, along with five + one = total six cookies.
Now that works well in browser. If I copy all the cookies from the browser into curl, the following call would work.
curl -X GET --header "Cookie: cookie1=value1; cookie2=value2; cookie3=value3; cookie4=value4; cookie5=value5; cookie6=value6" http://www.sitedomain.com/product_info.php?products_id=350799
However, how can I repeat this behavior in Google Chrome extension? Nothing I do seems to be working, yes I added the domain to the manifest.json and I've tried request, requestify (both browserified) and XMLHttpRequest. The requests seem fine (do include cookies) but the response I get seems to be the one I get when the site any receiving cookie.
manifest.json
{
"manifest_version": 2,
"name": "Getting started example",
"description": "This extension shows a Google Image search result for the current page",
"version": "1.0",
"browser_action": {
"default_icon": "icon.png",
"default_popup": "popup.html"
},
"permissions": [
"activeTab",
"cookies",
"https://sitedomain.com/*",
"http://sitedomain.com/*",
"https://www.sitedomain.com/*",
"http://www.sitedomain.com/*"
]
}
The code I use (request version):
var request = require('request');
request = request.defaults({jar: true});
var jar = request.jar();
var cookie1 = request.cookie('cookie1=value1');
var cookie2 = request.cookie('cookie2=value2');
var cookie3 = request.cookie('cookie3=value3');
var cookie4 = request.cookie('cookie4=value4');
var cookie5 = request.cookie('cookie5=value5');
var cookie6 = request.cookie('cookie6=value6');
var url = "http://www.sitedomain.com";
jar.setCookie(cookie1, url);
jar.setCookie(cookie2, url);
jar.setCookie(cookie3, url);
jar.setCookie(cookie4, url);
jar.setCookie(cookie5, url);
jar.setCookie(cookie6, url);
request({uri: http://www.sitedomain.com/product_info.php?products_id=350799, jar: jar}, function (error, response, html) {
if (!error && response.statusCode == 200) {
//breakpoint here and we did not receive the page after login
}
}
Any help would be appreciated.