I'm trying to call this API with the request module about 200-300 times with a Lambda function. I need to add second between each call so I don't get a 429 response. I've tried a few different ways to make this happen, but it seems to ignore the code to slow it down.
How do people normally slow down these requests in AWS lambda? It would be great if I could insert something like utilities.sleep(1000) in the loop to make it wait a second before continuing. I'm sure there is a simple solution to this issue, but all the examples I've seen seem to make it plex.
function findProjects(items){
var toggleData = [];
for( var i = 0; i < items.length; i++ ){
setTimeout( callToggle( items[i] ), 1000 );
}
function callToggle( data ){
request({
'url': '/' + data.toggle.data.id,
'method': 'GET',
'headers': {
'Content-Type': 'application/json',
'Accept': 'application/json'
},
'auth': {
'user': 'xxxxxxx',
'pass': 'api_token'
}}, function( error, response, body ){
if( error ) {
console.log( error );
context.done( null, error );
} else {
console.log(response.statusCode, "toggle projects were listed");
var info = JSON.parse(body);
toggleData.push(info);
}
});
}
findDocument( toggleData );
}
I'm trying to call this API with the request module about 200-300 times with a Lambda function. I need to add second between each call so I don't get a 429 response. I've tried a few different ways to make this happen, but it seems to ignore the code to slow it down.
How do people normally slow down these requests in AWS lambda? It would be great if I could insert something like utilities.sleep(1000) in the loop to make it wait a second before continuing. I'm sure there is a simple solution to this issue, but all the examples I've seen seem to make it plex.
function findProjects(items){
var toggleData = [];
for( var i = 0; i < items.length; i++ ){
setTimeout( callToggle( items[i] ), 1000 );
}
function callToggle( data ){
request({
'url': 'https://www.toggl./api/v8/projects/' + data.toggle.data.id,
'method': 'GET',
'headers': {
'Content-Type': 'application/json',
'Accept': 'application/json'
},
'auth': {
'user': 'xxxxxxx',
'pass': 'api_token'
}}, function( error, response, body ){
if( error ) {
console.log( error );
context.done( null, error );
} else {
console.log(response.statusCode, "toggle projects were listed");
var info = JSON.parse(body);
toggleData.push(info);
}
});
}
findDocument( toggleData );
}
-
1
Recursive function execution with
setTimeout()between calls – Krzysztof Safjanowski Commented Aug 31, 2016 at 18:37 -
i give you an example. At time
T0you iterate1000times and start 1000setTimeoutwith 1000ms of delay. I don't know how time pass between the first ant the 1000 call but very very little. Then you start 1000 times your method after exactly 1s. You should increment your time each time. ES:setTimeout(function, 1000+i)– gianlucatursi Commented Aug 31, 2016 at 18:41 - Maybe this is useful caolan.github.io/async/docs.html#.eachLimit – Ebrahim Pasbani Commented Aug 31, 2016 at 18:53
- 1 Don't do that: thedailywtf./articles/The-Speedup-Loop – Loïc Faure-Lacroix Commented Nov 25, 2016 at 17:23
4 Answers
Reset to default 1You can do something like this:
for(var i = 0; i<items.length; i++){
setTimeout(callToggl, 1000 + (( i * X ) % Y), items[i]);
}
Where Y is the max delay (1000 + Y) then you want (es 5 sec) and X is the timing for each call (es X=10 : 1000,1010,1020,1030,...
if you want 1s each call:
for(var i = 0; i<items.length; i++){
setTimeout(callToggl(items[i]), 1000 + ( i * 1000 ));
}
EDIT
for(var i = 0; i<items.length; i++){
setTimeout(callToggl, 1000 + ( i * 1000 ), items[i]);
}
As you probably know, javascript code does not block on io (unless using specific sync api's which block your entire code which is a bad practice that should be avoided unless you have a really good reason to block the entire code(loading config files on startup... etc...))
so what you need to do, is simply wait for the response
it used to be a bit plicated in the past to orchestrate it all, but currently, using --harmony flag (to activate latest js features in node) you can use the shiny new async functions syntax. (await/async)
the function you are running inside must be declared async, then after each iteration, you need to wait for the response of that http call using "await" keyword. this keyword makes the code seem as if its blocking and waiting for the resolved answer, although its not.
i used "fetch" instead of "request" because it plays nice with async functions (being promise based) but you can use any other method as long as you return a promise. (you can even promisify your existing callback based api with Promise object, but it will make everything look uglier, so please don't :) )
here is the modified code. I'm not really sure if it works as is. But the idea is pretty clear i think.
On any case, its a great opportunity for you to learn how to work with async functions in case you didn't use them already, they really make life easier.
//making enture function async, so you can use 'await' inside it
async function findProjects(items){
var toggleData = [];
for( var i = 0; i < items.length; i++ ){
//setTimeout( callToggle( items[i] ), 1000 );
//instead of using a timeout, you need to wait for response before continuing to next iteration
await response = callToggle(items[i]);
toggleData.push(JSON.parse(response.body));
}
async function callToggle( data ){
/*request({
'url': 'https://www.toggl./api/v8/projects/' + data.toggle.data.id,
'method': 'GET',
'headers': {
'Content-Type': 'application/json',
'Accept': 'application/json'
},
'auth': {
'user': 'xxxxxxx',
'pass': 'api_token'
}}, function( error, response, body ){
if( error ) {
console.log( error );
context.done( null, error );
} else {
console.log(response.statusCode, "toggle projects were listed");
var info = JSON.parse(body);
toggleData.push(info);
}
});*/
// to make things simpler, use fetch instead of request which is promise based
var myInit = { 'method': 'GET',
'headers': {
'Content-Type': 'application/json',
'Accept': 'application/json'
},
'auth': {
'user': 'xxxxxxx',
'pass': 'api_token'
}
};
return fetch("https://www.toggl./api/v8/projects/",myInit);
}
findDocument( toggleData );
}
You can chain the requests together:
function findProjects(items){
var toggleData = [];
// chain requests with delay
items.reduce(function (requestChain, item) {
return requestChain
.then(callToggle.bind(null, item))
.then(wait.bind(null, 1000));
}, Promise.resolve());
function wait (ms) {
return new Promise(function (resolve, reject) {
setTimeout(resolve, ms);
});
}
function callToggle(data) {
request({
'url': 'https://www.toggl./api/v8/projects/' + data.toggle.data.id,
'method': 'GET',
'headers': {
'Content-Type': 'application/json',
'Accept': 'application/json'
},
'auth': {
'user': 'xxxxxxx',
'pass': 'api_token'
}}, function( error, response, body ){
if( error ) {
console.log( error );
context.done( null, error );
} else {
console.log(response.statusCode, "toggle projects were listed");
var info = JSON.parse(body);
toggleData.push(info);
}
});
}
findDocument( toggleData );
}
While Node.js is single-threaded, setTimeout does not create a single stack of synchronous calls. When you use your for loop you immediately set each subsequent call to 1000ms ahead, so they all activate at roughly the same time. Instead, you likely want to use a third-party promise lib to wait.
You could do something as simple as:
const Bluebird = require('bluebird');
const $http = require('http-as-promised');
const _ = require('lodash');
const timeout = 1000;
const items = [];
const headers = { 'Content-Type': 'application/json', 'Accept': 'application/json' };
const auth = { 'user': 'xxxxxxx', 'pass': 'api_token' };
const makeRequest = (id) => {
const url = 'https://www.toggl./api/v8/projects/' + id;
return $http.get({ url, headers, auth });
};
const waitCall = (data) => {
return Bluebird
.resolve(makeRequest(data.toggl.data.id))
.wait(timeout);
};
Bluebird.mapSeries(items, waitCall);
http://bluebirdjs./docs/api/promise.mapseries.html