This is driving me nuts!
I have got an axios call returning an json array object in console - Check!
>(100) [{…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}]
>0:
id: 54892250
iid: 1001
ref: "master"
sha: "63e3fc014e207dd4e708749cee3e89a1aa416b7d"
created_at: "2020-03-05T18:57:02.793Z"
updated_at: "2020-03-05T19:06:27.651Z"
>user: {id: someuserid, name: "someuser", username: "someusername", state: "active", avatar_url: ".png", …}
>environment: {id: 123456, name: "somename", slug: "somename-iw5iqp", external_url: ""}
>deployable: {id: someid, status: "success", stage: "deploy", name: "deploy-staging", ref: "master", …}
status: "success"
__proto__: Object
>1: {id: 54804365, iid: 1000, ref: "filter-out-expired-items", sha: "6225d8018c86fa906063aeea5c10c710ddced14c", created_at: "2020-03-05T12:25:18.949Z", …}
>2: {id: 54804175, iid: 999, ref: "master", sha: "aa5e50c50ba95e9b16cbc57fb968bf9075c625b2", created_at: "2020-03-05T12:24:02.284Z", …}
>3: {id: 54801934, iid: 998, ref: "filter-out-expired-items", sha: "4fc2
Now I need to do 2 things with this result:
- when fully expanded this response is large - and there are thousands of them. Rather than store 1000s of giant json reponses in a vuex state - I want to first only extract the values we care about which are the following:
- id
- ref
- environment.name
- status
- deployable.tag
to something like:
{ "id": id, "ref": ref, "environment": environment.name, "status": status, "tag": deployable.tag, }
How do I do this?
- This response is paginated over many pages above is just a partial sample from one page's results.
I have a working loop for that and I am getting the console.log for all of them. BUT what I need to do is concatenate all pages (now stripped to the above fields) before I mit to the state. If have tried object copies, pushing to arrays and all kinds of utils promising the world - but I cannot get any of them working :)
So in summary:
- get a page's results
- squash item in the array to a simplified version
- collect them all into a object I can insert into the state
Relevant part of the loop
// Use headers to get the number of pages
const pages = await axios.head(url, options)
.then((response) => response.headers['x-total-pages']);
console.log('pages=', pages); // DEBUG
// Loop through and push them to an array
for (let i = 0; i <= pages; i += 1) {
console.log('i=', i);
options = {
headers: {
'Private-Token': token,
},
params: {
order_by: 'created_at',
sort: 'desc',
per_page: 100,
page: i,
},
};
axios.get(url, options)
.then((result) => { console.log(result.data); })
}
This is driving me nuts!
I have got an axios call returning an json array object in console - Check!
>(100) [{…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}, {…}]
>0:
id: 54892250
iid: 1001
ref: "master"
sha: "63e3fc014e207dd4e708749cee3e89a1aa416b7d"
created_at: "2020-03-05T18:57:02.793Z"
updated_at: "2020-03-05T19:06:27.651Z"
>user: {id: someuserid, name: "someuser", username: "someusername", state: "active", avatar_url: "https://assets.gitlab-static/uploads/-/system/user/avatar/4534925/avatar.png", …}
>environment: {id: 123456, name: "somename", slug: "somename-iw5iqp", external_url: "https://someaddress."}
>deployable: {id: someid, status: "success", stage: "deploy", name: "deploy-staging", ref: "master", …}
status: "success"
__proto__: Object
>1: {id: 54804365, iid: 1000, ref: "filter-out-expired-items", sha: "6225d8018c86fa906063aeea5c10c710ddced14c", created_at: "2020-03-05T12:25:18.949Z", …}
>2: {id: 54804175, iid: 999, ref: "master", sha: "aa5e50c50ba95e9b16cbc57fb968bf9075c625b2", created_at: "2020-03-05T12:24:02.284Z", …}
>3: {id: 54801934, iid: 998, ref: "filter-out-expired-items", sha: "4fc2
Now I need to do 2 things with this result:
- when fully expanded this response is large - and there are thousands of them. Rather than store 1000s of giant json reponses in a vuex state - I want to first only extract the values we care about which are the following:
- id
- ref
- environment.name
- status
- deployable.tag
to something like:
{ "id": id, "ref": ref, "environment": environment.name, "status": status, "tag": deployable.tag, }
How do I do this?
- This response is paginated over many pages above is just a partial sample from one page's results.
I have a working loop for that and I am getting the console.log for all of them. BUT what I need to do is concatenate all pages (now stripped to the above fields) before I mit to the state. If have tried object copies, pushing to arrays and all kinds of utils promising the world - but I cannot get any of them working :)
So in summary:
- get a page's results
- squash item in the array to a simplified version
- collect them all into a object I can insert into the state
Relevant part of the loop
// Use headers to get the number of pages
const pages = await axios.head(url, options)
.then((response) => response.headers['x-total-pages']);
console.log('pages=', pages); // DEBUG
// Loop through and push them to an array
for (let i = 0; i <= pages; i += 1) {
console.log('i=', i);
options = {
headers: {
'Private-Token': token,
},
params: {
order_by: 'created_at',
sort: 'desc',
per_page: 100,
page: i,
},
};
axios.get(url, options)
.then((result) => { console.log(result.data); })
}
- can you post your sub arrays (page array/list) structure – EugenSunic Commented Mar 7, 2020 at 21:32
- It's Gitlab's API I am working with - and they have good docs and sample responses The deployments api docs.gitlab./ee/api/… Their pagination docs.gitlab./ee/api/README.html#pagination – Seer Commented Mar 7, 2020 at 21:46
- And I am struggling to formulate the goal succinctly I know :) But basically its a summary of deployment stats over a year. For which their can be 3000-4000 results per year so thats why I think its important to fiter the dataset before throwing it into the state – Seer Commented Mar 7, 2020 at 21:49
-
When you're doing all requests async, you'll loose the Sorting, and you won't really know when all requests are done. You could use
Promise.all()when all of request promises have resolved. Is that what you want? – KeitelDOG Commented Mar 7, 2020 at 22:49
3 Answers
Reset to default 2Create an array for the total results:
const results = [];
Where you're logging, parse each individual result:
result.data.forEach(item => {
results.push({
id: item.id,
ref: item.ref,
environment: item.environment.name,
status: item.status,
tag: item.deployable.tag
});
});
Note that it may be a performance optimization to use a normal for loop instead of forEach.
I think you could use Promise to keep the Order of requests, hence the Sorting in across pages data. The key is to put each axios call into an array, and then call Promise.all([]) with that array to get all reponses all at once in the original order you made the requests.
// Use headers to get the number of pages
const pages = await axios.head(url, options)
.then((response) => response.headers['x-total-pages']);
console.log('pages=', pages); // DEBUG
let promises = [];
// Loop through and push them to an array
for (let i = 0; i <= pages; i += 1) {
console.log('i=', i);
options = {
headers: {
'Private-Token': token,
},
params: {
order_by: 'created_at',
sort: 'desc',
per_page: 100,
page: i,
},
};
let promise = axios.get(url, options);
// this below keep responses order
promises.push(promise);
}
// This wait for all Axios calls to be resolved as promise
Promise.all(promises)
.then((result) => {
// now if you have 10 pages, you'll have result[0] to result[9], each of them is an axios response
console.log(result[0].data);
console.log(result[1].data); // if pages > 0
let items = []; // you can declare it outside too for Scope access
for (let i = 0; i <= pages; i += 1) {
// so many records, then take the minimum info
if (result[i].data.length >= 1000) {
result[i].data.forEach(item => {
items.push({
id: item.id,
ref: item.ref,
environment: item.environment.name,
status: item.status,
tag: item.deployable.tag
});
}
} else {
// Not so many records then take all info
items = items.concat(result[i]);
}
}
// TODO: do whatever you want with the items array now
});
I would use Array.prototype.map here to map the original objects into simplified ones with only a subset of the properties:
const newResult = result.data.map(d => ({
id: d.id,
ref: d.ref,
environmentName: d.environment.name,
status: d.deployable.status,
deployableTag: d.deployable.tag
}))
const data = [
{
"created_at": "2016-08-11T07:36:40.222Z",
"updated_at": "2016-08-11T07:38:12.414Z",
"deployable": {
"mit": {
"author_email": "[email protected]",
"author_name": "Administrator",
"created_at": "2016-08-11T09:36:01.000+02:00",
"id": "99d03678b90d914dbb1b109132516d71a4a03ea8",
"message": "Merge branch 'new-title' into 'master'\r\n\r\nUpdate README\r\n\r\n\r\n\r\nSee merge request !1",
"short_id": "99d03678",
"title": "Merge branch 'new-title' into 'master'\r"
},
"coverage": null,
"created_at": "2016-08-11T07:36:27.357Z",
"finished_at": "2016-08-11T07:36:39.851Z",
"id": 657,
"name": "deploy",
"ref": "master",
"runner": null,
"stage": "deploy",
"started_at": null,
"status": "success",
"tag": false,
"user": {
"id": 1,
"name": "Administrator",
"username": "root",
"state": "active",
"avatar_url": "http://www.gravatar./avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon",
"web_url": "http://gitlab.dev/root",
"created_at": "2015-12-21T13:14:24.077Z",
"bio": null,
"location": null,
"public_email": "",
"skype": "",
"linkedin": "",
"twitter": "",
"website_url": "",
"organization": ""
},
"pipeline": {
"created_at": "2016-08-11T02:12:10.222Z",
"id": 36,
"ref": "master",
"sha": "99d03678b90d914dbb1b109132516d71a4a03ea8",
"status": "success",
"updated_at": "2016-08-11T02:12:10.222Z",
"web_url": "http://gitlab.dev/root/project/pipelines/12"
}
},
"environment": {
"external_url": "https://about.gitlab.",
"id": 9,
"name": "production"
},
"id": 41,
"iid": 1,
"ref": "master",
"sha": "99d03678b90d914dbb1b109132516d71a4a03ea8",
"user": {
"avatar_url": "http://www.gravatar./avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon",
"id": 1,
"name": "Administrator",
"state": "active",
"username": "root",
"web_url": "http://localhost:3000/root"
}
},
{
"created_at": "2016-08-11T11:32:35.444Z",
"updated_at": "2016-08-11T11:34:01.123Z",
"deployable": {
"mit": {
"author_email": "[email protected]",
"author_name": "Administrator",
"created_at": "2016-08-11T13:28:26.000+02:00",
"id": "a91957a858320c0e17f3a0eca7cfacbff50ea29a",
"message": "Merge branch 'rename-readme' into 'master'\r\n\r\nRename README\r\n\r\n\r\n\r\nSee merge request !2",
"short_id": "a91957a8",
"title": "Merge branch 'rename-readme' into 'master'\r"
},
"coverage": null,
"created_at": "2016-08-11T11:32:24.456Z",
"finished_at": "2016-08-11T11:32:35.145Z",
"id": 664,
"name": "deploy",
"ref": "master",
"runner": null,
"stage": "deploy",
"started_at": null,
"status": "success",
"tag": false,
"user": {
"id": 1,
"name": "Administrator",
"username": "root",
"state": "active",
"avatar_url": "http://www.gravatar./avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon",
"web_url": "http://gitlab.dev/root",
"created_at": "2015-12-21T13:14:24.077Z",
"bio": null,
"location": null,
"public_email": "",
"skype": "",
"linkedin": "",
"twitter": "",
"website_url": "",
"organization": ""
},
"pipeline": {
"created_at": "2016-08-11T07:43:52.143Z",
"id": 37,
"ref": "master",
"sha": "a91957a858320c0e17f3a0eca7cfacbff50ea29a",
"status": "success",
"updated_at": "2016-08-11T07:43:52.143Z",
"web_url": "http://gitlab.dev/root/project/pipelines/13"
}
},
"environment": {
"external_url": "https://about.gitlab.",
"id": 9,
"name": "production"
},
"id": 42,
"iid": 2,
"ref": "master",
"sha": "a91957a858320c0e17f3a0eca7cfacbff50ea29a",
"user": {
"avatar_url": "http://www.gravatar./avatar/e64c7d89f26bd1972efa854d13d7dd61?s=80&d=identicon",
"id": 1,
"name": "Administrator",
"state": "active",
"username": "root",
"web_url": "http://localhost:3000/root"
}
}
]
const result = data.map(d => ({
id: d.id,
ref: d.ref,
environmentName: d.environment.name,
status: d.deployable.status,
deployableTag: d.deployable.tag
}))
console.log(result)