Here is my code so far:

const allRows = [];

async function fileToLines(file) {
  return new Promise((resolve, reject) => {
    reader = new FileReader();
    reader.onload = function(e) {
      parsedLines = e.target.result.split(/\r|\n|\r\n/);
      resolve(parsedLines);
    };
    reader.readAsText(file);
  });
}

document
.getElementById('fileInput')
.addEventListener('change', async function(e) {
  var file = e.target.files[0];

  if (file != undefined) {
    fileToLines(file).then( async id => {
      console.log(id)
      console.log(parsedLines)
      console.log(typeof id);

      var idInt = id.map(Number);
      var idFiltered = id.filter(function(v){return v!==''});

      console.log(idFiltered)

      for(let id of idFiltered) {
        const row = await getRelease(id);
        allRows.push(row);
      }
      download();
    });
  }
});

function getRelease(idFiltered) {
  return fetch(`https://api.***/releases/${idFiltered}`, {
    headers: {
    'User-Agent': '***/0.1',
    },
  })
  .then(response => response.json())
  .then(data => {
    if (data.message === 'Release not found.') {
      return { error: `Release with ID ${idFiltered} does not exist` };
    } else {
      const id = data.id;
      const delimiter = document.getElementById("delimiter").value || "|";
      const artists = data.artists ? data.artists.map(artist => artist.name) : [];
      const barcode = data.identifiers.filter(id => id.type === 'Barcode')
      .map(barcode => barcode.value);
      var formattedBarcode = barcode.join(delimiter);
      const country = data.country || 'Unknown';
      const genres = data.genres || [];
      const formattedGenres = genres.join(delimiter);
      const labels = data.labels ? data.labels.map(label => label.name) : [];
      const formattedLabels = labels.join(delimiter);
      const catno = data.labels ? data.labels.map(catno => catno.catno) : [];
      const formattedCatNo = catno.join(delimiter);
      const styles = data.styles || [];
      const formattedStyles = styles.join(delimiter);
      const tracklist = data.tracklist ? data.tracklist
      .map(track => track.title) : [];
      const formattedTracklist = tracklist.join(delimiter);
      const year = data.year || 'Unknown';
      const format = data.formats ? data.formats.map(format => format.name) : [];
      const qty = data.formats ? data.formats.map(format => format.qty) : [];
      const descriptions = data.formats ? data.formats
      .map(descriptions => descriptions.descriptions) : [];
      const preformattedDescriptions = descriptions.toString()
      .replace('"','""').replace(/,/g, ', ');
      const formattedDescriptions = '"' + preformattedDescriptions + '"';

      return [idFiltered,
        artists,
        format,
        qty,
        formattedDescriptions,
        formattedLabels,
        formattedCatNo,
        country,
        year,
        formattedGenres,
        formattedStyles,
        formattedBarcode,
        formattedTracklist
      ];
    }
  });
}

function download() {
  const ROW_NAMES = [
    "release_id",
    "artist",
    "format",
    "qty",
    "format descriptions",
    "label",
    "catno",
    "country",
    "year",
    "genres",
    "styles",
    "barcode",
    "tracklist"
  ];
  var csvContent = "data:text/csv;charset=utf-8,"
  + ROW_NAMES + "\n" + allRows.map(e => e.join(",")).join("\n");

  console.log(csvContent);

  var encodedUri = encodeURI(csvContent);
  var link = document.createElement("a");
  link.setAttribute("href", encodedUri);
  link.setAttribute("download", "my_data.csv");
  document.body.appendChild(link); // Required for FF
  link.click();
}

When I was previously trying to crack this problem 2.5 years ago (!) someone told me the easiest way "is to maintain a chain of promises to keep track of the requests", like this...

  const timer = ms => new Promise(resolve => setTimeout(resolve, ms));

  let requests = Promise.resolve();

  function getRelease(id) {
   const apiCall = requests.then(() =>
    fetch(`https://api.***/releases/${id}`, {
      headers: {
        'User-Agent': '***/0.1',
      }
    })   
   );

   // add to chain / queue 
   requests = apiCall.then(response => 
    +response.headers.get("X-***-Ratelimit-Remaining") <= 1 && timer(60 * 1000)
   );

   return apiCall
     .then(response => response.json())
     .then(parseReleaseData);
  }

The person who suggested this code mented...

Now one request will be done after another, and if the rate limit gets reached it waits a minute.

You might want to retry in case of a rate limiting error. You could also add multiple promise queues to allow for higher throughput.

It seems that when I tried that previously, it set a 60s delay before making any calls? I think I would like to try this method again, but I'm not sure how to code it. Like, I'm not sure how const apiCall = requests.then(() => would fit in with my current code. I can see that the suggested code actually returns 'apiCall', whereas my method is set up to return all the individual data fields, so I'm not sure how to proceed there. It seems like a good method to get the Ratelimit from the host and set a timeout as needed, but I'm just not sure where to start really. Any help please?

Edit: I've been trying to do it like this, but it still doesn't work:

const timer = ms => new Promise(resolve => setTimeout(resolve, ms));

const createThrottler = (rateLimit) => {
  let requestTimestamp = 0;
  return (requestHandler) => {
    return async (...params) => {
      const currentTimestamp = Math.floor(Date.now() / 1000);
      if (currentTimestamp < requestTimestamp + rateLimit) {
        await timer(rateLimit - (currentTimestamp - requestTimestamp))
      }
      requestTimestamp = Math.floor(Date.now() / 1000);
      return await requestHandler(...params);
    }
  }
}

const throttle = createThrottler(2500);

const throttleFetch = throttle(fetch);

Edit2: I was wondering if there was a problem that I had this line mented out:

const rateLimit = Math.floor((60 / response.headers.get("X-Discogs-Ratelimit-Remaining")) * 1000);

So I tried un-menting it, but now I get

Uncaught ReferenceError: response is not defined

Edit3: I got a suggestion that get the createThrottler() function working:-

const rateLimit = 2500;

const timer = ms => new Promise(resolve => setTimeout(resolve, ms));

const createThrottler = (rateLimit) => {
  let requestTimestamp = 0;
  return (requestHandler) => {
    return async (...params) => {
      const currentTimestamp = Number(Date.now());
      if (currentTimestamp < requestTimestamp + rateLimit) {
        const timeOut = rateLimit - (currentTimestamp - requestTimestamp);
        requestTimestamp = Number(Date.now()) + timeOut;
        await timer(timeOut)
      }
      requestTimestamp = Number(Date.now());
      return await requestHandler(...params);
    }
  }
}

Can't say I would ever have worked that out for myself, but there we are. So now I am trying to work out how and where to code

const rateLimit = Math.floor((60 / response.headers.get("X-Discogs-Ratelimit-Remaining")) * 1000);

without getting

Uncaught (in promise) ReferenceError: response is not defined

Here is my code so far:

const allRows = [];

async function fileToLines(file) {
  return new Promise((resolve, reject) => {
    reader = new FileReader();
    reader.onload = function(e) {
      parsedLines = e.target.result.split(/\r|\n|\r\n/);
      resolve(parsedLines);
    };
    reader.readAsText(file);
  });
}

document
.getElementById('fileInput')
.addEventListener('change', async function(e) {
  var file = e.target.files[0];

  if (file != undefined) {
    fileToLines(file).then( async id => {
      console.log(id)
      console.log(parsedLines)
      console.log(typeof id);

      var idInt = id.map(Number);
      var idFiltered = id.filter(function(v){return v!==''});

      console.log(idFiltered)

      for(let id of idFiltered) {
        const row = await getRelease(id);
        allRows.push(row);
      }
      download();
    });
  }
});

function getRelease(idFiltered) {
  return fetch(`https://api.***./releases/${idFiltered}`, {
    headers: {
    'User-Agent': '***/0.1',
    },
  })
  .then(response => response.json())
  .then(data => {
    if (data.message === 'Release not found.') {
      return { error: `Release with ID ${idFiltered} does not exist` };
    } else {
      const id = data.id;
      const delimiter = document.getElementById("delimiter").value || "|";
      const artists = data.artists ? data.artists.map(artist => artist.name) : [];
      const barcode = data.identifiers.filter(id => id.type === 'Barcode')
      .map(barcode => barcode.value);
      var formattedBarcode = barcode.join(delimiter);
      const country = data.country || 'Unknown';
      const genres = data.genres || [];
      const formattedGenres = genres.join(delimiter);
      const labels = data.labels ? data.labels.map(label => label.name) : [];
      const formattedLabels = labels.join(delimiter);
      const catno = data.labels ? data.labels.map(catno => catno.catno) : [];
      const formattedCatNo = catno.join(delimiter);
      const styles = data.styles || [];
      const formattedStyles = styles.join(delimiter);
      const tracklist = data.tracklist ? data.tracklist
      .map(track => track.title) : [];
      const formattedTracklist = tracklist.join(delimiter);
      const year = data.year || 'Unknown';
      const format = data.formats ? data.formats.map(format => format.name) : [];
      const qty = data.formats ? data.formats.map(format => format.qty) : [];
      const descriptions = data.formats ? data.formats
      .map(descriptions => descriptions.descriptions) : [];
      const preformattedDescriptions = descriptions.toString()
      .replace('"','""').replace(/,/g, ', ');
      const formattedDescriptions = '"' + preformattedDescriptions + '"';

      return [idFiltered,
        artists,
        format,
        qty,
        formattedDescriptions,
        formattedLabels,
        formattedCatNo,
        country,
        year,
        formattedGenres,
        formattedStyles,
        formattedBarcode,
        formattedTracklist
      ];
    }
  });
}

function download() {
  const ROW_NAMES = [
    "release_id",
    "artist",
    "format",
    "qty",
    "format descriptions",
    "label",
    "catno",
    "country",
    "year",
    "genres",
    "styles",
    "barcode",
    "tracklist"
  ];
  var csvContent = "data:text/csv;charset=utf-8,"
  + ROW_NAMES + "\n" + allRows.map(e => e.join(",")).join("\n");

  console.log(csvContent);

  var encodedUri = encodeURI(csvContent);
  var link = document.createElement("a");
  link.setAttribute("href", encodedUri);
  link.setAttribute("download", "my_data.csv");
  document.body.appendChild(link); // Required for FF
  link.click();
}

When I was previously trying to crack this problem 2.5 years ago (!) someone told me the easiest way "is to maintain a chain of promises to keep track of the requests", like this...

  const timer = ms => new Promise(resolve => setTimeout(resolve, ms));

  let requests = Promise.resolve();

  function getRelease(id) {
   const apiCall = requests.then(() =>
    fetch(`https://api.***./releases/${id}`, {
      headers: {
        'User-Agent': '***/0.1',
      }
    })   
   );

   // add to chain / queue 
   requests = apiCall.then(response => 
    +response.headers.get("X-***-Ratelimit-Remaining") <= 1 && timer(60 * 1000)
   );

   return apiCall
     .then(response => response.json())
     .then(parseReleaseData);
  }

The person who suggested this code mented...

Now one request will be done after another, and if the rate limit gets reached it waits a minute.

You might want to retry in case of a rate limiting error. You could also add multiple promise queues to allow for higher throughput.

It seems that when I tried that previously, it set a 60s delay before making any calls? I think I would like to try this method again, but I'm not sure how to code it. Like, I'm not sure how const apiCall = requests.then(() => would fit in with my current code. I can see that the suggested code actually returns 'apiCall', whereas my method is set up to return all the individual data fields, so I'm not sure how to proceed there. It seems like a good method to get the Ratelimit from the host and set a timeout as needed, but I'm just not sure where to start really. Any help please?

Edit: I've been trying to do it like this, but it still doesn't work:

const timer = ms => new Promise(resolve => setTimeout(resolve, ms));

const createThrottler = (rateLimit) => {
  let requestTimestamp = 0;
  return (requestHandler) => {
    return async (...params) => {
      const currentTimestamp = Math.floor(Date.now() / 1000);
      if (currentTimestamp < requestTimestamp + rateLimit) {
        await timer(rateLimit - (currentTimestamp - requestTimestamp))
      }
      requestTimestamp = Math.floor(Date.now() / 1000);
      return await requestHandler(...params);
    }
  }
}

const throttle = createThrottler(2500);

const throttleFetch = throttle(fetch);

Edit2: I was wondering if there was a problem that I had this line mented out:

const rateLimit = Math.floor((60 / response.headers.get("X-Discogs-Ratelimit-Remaining")) * 1000);

So I tried un-menting it, but now I get

Uncaught ReferenceError: response is not defined

Edit3: I got a suggestion that get the createThrottler() function working:-

const rateLimit = 2500;

const timer = ms => new Promise(resolve => setTimeout(resolve, ms));

const createThrottler = (rateLimit) => {
  let requestTimestamp = 0;
  return (requestHandler) => {
    return async (...params) => {
      const currentTimestamp = Number(Date.now());
      if (currentTimestamp < requestTimestamp + rateLimit) {
        const timeOut = rateLimit - (currentTimestamp - requestTimestamp);
        requestTimestamp = Number(Date.now()) + timeOut;
        await timer(timeOut)
      }
      requestTimestamp = Number(Date.now());
      return await requestHandler(...params);
    }
  }
}

Can't say I would ever have worked that out for myself, but there we are. So now I am trying to work out how and where to code

const rateLimit = Math.floor((60 / response.headers.get("X-Discogs-Ratelimit-Remaining")) * 1000);

without getting

Uncaught (in promise) ReferenceError: response is not defined

• javascript
• json
• csv

• Use generators functions with lodash debounce. – Umair Riaz Commented Jan 14, 2022 at 12:43

4 Answers 4

Have you looked at debounce?

You can rate limit as in 1 call serviced in any defined period. Think about this as quantizing. The other way is to count calls over an extended time-frame and then either block further calls indefinitely or for a defined duration - it es down to your preferred use-case.

Normally rate-limiting has more to do with security and the first option (1 call serviced in a defined period) is apt. If you are doing this for a web API, you may wish to reject requests that are 'too soon' and give the requestor some type of feedback with an appropriate HTTP status code.

How to implement all the different options is discussed here: https://thoughtspile.github.io/2018/07/07/rate-limit-promises/

EDIT: In response to OP ment below and reviewing the code... I think you're overthinking it.

FWIW I use debounce for the most part (equivalent to your "throttle") and it is literally used along the lines of debounce(functionReference,timeoutInMilliseconds).

The code looks like this

function debounce(func, waitFor) {
    let timeout;
    return (...args) => new Promise(resolve => {
        if (timeout) {
            clearTimeout(timeout);
        }
        timeout = setTimeout(() => resolve(func(...args)), waitFor);
    });
}

Change your throttle(fetch) to my debounce(fetch,2500) and it should be enough. You don't need to have the assignment operation on that line, just call it, or write another function called debouncedFetch to encapsulate it and just call that from wherever you need to.

I am trying to solve a very similar problem using the same API endpoint as OP. My solution gets the full release info of an array of release IDs. I doubt my solution is anywhere near optimal, however it works ok.

const recordIDs: number[] = JSON.parse(req.body.records)
const endpoint = `${discogsAPIURL}releases/`
const user = req.user! as IUser
const records: ReleaseFull[] = []
const throttlePoint = 6 // X-Discogs-Ratelimit-Remaining to begin throttling
let requestsMade = 0
let limitRemaining = 60
let wait = 0

while (requestsMade < recordIDs.length) {
  if (wait) await new Promise((resolve) => setTimeout(resolve, wait))
  const url = endpoint + recordIDs[requestsMade].toString()
  const response = await authorisedDiscogsRequest(url, user)

  if (response.status === 200) {
    requestsMade++
    const retrievedRecord = (await response.json()) as ReleaseFull
    records.push(retrievedRecord)
    res.write("data: " + `${requestsMade / recordIDs.length}\n\n`)
    limitRemaining = parseInt(
      response.headers.get("X-Discogs-Ratelimit-Remaining") || "0"
    )
    wait =
      limitRemaining < throttlePoint
        ? (throttlePoint - limitRemaining) * 1000
        : 0
  } else if (response.status === 429) {
    wait = wait + 10000
  } else if (response.status === 404) {
    res.write("data: " + `Error: A release was not found by Discogs.\n\n`)
    res.end()
  } else {
    res.write("data: " + `Error: Unexpected error.\n\n`)
    res.end()
  }
}

Try promise-ratelimit

from their docs:

var throttle = require('promise-ratelimit')(2000); /* rateInMilliseconds */

var startTime = Date.now();

for (var i = 0; i < 10; i++) {
    throttle().then(function() { console.log(Date.now() - startTime); });
}

It's better to use a fetch wrapper with plugins support, here we use xior.js, the example:

import xior from 'xior';
import throttlePlugin from 'xior/plugins/throttle';

const http = xior.create();

http.plugins.use(
  throttlePlugin({
    onThrottle(config) {
      console.log(`Throttle requests ${config.method} ${config.url}`);
    },
  })
);

There are more useful plugins in xior.js. check -> https://github./suhaotian/xior