Motive : Transfer Files of around 10gb to 50gb from client to server using node
Problem : Consuming a lot of memory sometimes reaches around 6gb and system hangs
What I want is : my code should not use more than 200mb of ram..
What I Am Doing : I pause my stream when ram usage reaches 200 mb
Expected : That I am gonna pause streaming data when ram usage reaches 200 mb and will Resume it when usage es down.
What's happening : I pauses when usage is above 200 mb but it does not resumes cos even the script is pause the ram usage is not ming down.
Client.js
var net = require('net'),
fs = require('fs'),
path = require('path');
socket = new net.Socket();
socket.connect(6000, 127.0.0.1);
socket.on('connect',function(){
// I am reading around 10 gb of file in chunks
var readStream = fs.createReadStream("File Name", {highWaterMark: 16384});
// Checking ram usage every second to ensure it does not consume more than 200 mb of ram, If i do not write this check it even uses 4gb+ ram for this much big file and hangs my node script.
setInterval(function(){
if(process.memoryUsage().rss > 209715200){
// if ram consumtion is more that 200 mb
console.log("Pause");
global.gc();
readStream.pause();
}else{
readStream.on('pause',function(){
readStream.resume();
});
}
},1000);
readStream.on('data', function(chunk){
console.log("Used Mem "+process.memoryUsage().rss);
var head = new Buffer.from("FILE");
var sizeHex = chunk.length.toString(16);
while(sizeHex.length < 4){
sizeHex = "0" + sizeHex;
}
var size = new Buffer.from(sizeHex);
var delimiter = new Buffer.from("@");
var pack = Buffer.concat([head, size, chunk, delimiter]);
// sending data to server
// This sending part start consuming ram
socket.write(pack,function(){
});
});
readStream.on('close', function(){
socket.end();
global.gc();
});
});
Server.js
var net = require('net'),
fs = require('fs'),
path = require('path');
var server = net.createServer(function(socket){
var packets = 0;
var buffer = new Buffer.alloc(0);
// Receiving Data
socket.on('data', function(chunk){
buffer = Buffer.concat([buffer, chunk]);
});
// when Client socket ends write file on server
socket.on('close', function(){
var writeStream = fs.createWriteStream("New File Name");
while(buffer.length){
var head = buffer.slice(0, 4);
if(head.toString() != "FILE"){
console.log("ERROR!!!!");
process.exit(1);
}
var sizeHex = buffer.slice(4, 8);
var size = parseInt(sizeHex, 16);
var content = buffer.slice(8, size + 8);
var delimiter = buffer.slice(size + 8, size + 9);
if(delimiter != "@"){
console.log("wrong delimiter!!!");
process.exit(1);
}
writeStream.write(content);
buffer = buffer.slice(size + 9);
}
setTimeout(function(){
writeStream.end();
}, 2000);
});
});
server.listen(6000);
Ram Usage in system Monitor
Before Running Above Script : 1.6gb of 6 gb
After Running Above Script : 1.8 gb of 6gb
Motive : Transfer Files of around 10gb to 50gb from client to server using node
Problem : Consuming a lot of memory sometimes reaches around 6gb and system hangs
What I want is : my code should not use more than 200mb of ram..
What I Am Doing : I pause my stream when ram usage reaches 200 mb
Expected : That I am gonna pause streaming data when ram usage reaches 200 mb and will Resume it when usage es down.
What's happening : I pauses when usage is above 200 mb but it does not resumes cos even the script is pause the ram usage is not ming down.
Client.js
var net = require('net'),
fs = require('fs'),
path = require('path');
socket = new net.Socket();
socket.connect(6000, 127.0.0.1);
socket.on('connect',function(){
// I am reading around 10 gb of file in chunks
var readStream = fs.createReadStream("File Name", {highWaterMark: 16384});
// Checking ram usage every second to ensure it does not consume more than 200 mb of ram, If i do not write this check it even uses 4gb+ ram for this much big file and hangs my node script.
setInterval(function(){
if(process.memoryUsage().rss > 209715200){
// if ram consumtion is more that 200 mb
console.log("Pause");
global.gc();
readStream.pause();
}else{
readStream.on('pause',function(){
readStream.resume();
});
}
},1000);
readStream.on('data', function(chunk){
console.log("Used Mem "+process.memoryUsage().rss);
var head = new Buffer.from("FILE");
var sizeHex = chunk.length.toString(16);
while(sizeHex.length < 4){
sizeHex = "0" + sizeHex;
}
var size = new Buffer.from(sizeHex);
var delimiter = new Buffer.from("@");
var pack = Buffer.concat([head, size, chunk, delimiter]);
// sending data to server
// This sending part start consuming ram
socket.write(pack,function(){
});
});
readStream.on('close', function(){
socket.end();
global.gc();
});
});
Server.js
var net = require('net'),
fs = require('fs'),
path = require('path');
var server = net.createServer(function(socket){
var packets = 0;
var buffer = new Buffer.alloc(0);
// Receiving Data
socket.on('data', function(chunk){
buffer = Buffer.concat([buffer, chunk]);
});
// when Client socket ends write file on server
socket.on('close', function(){
var writeStream = fs.createWriteStream("New File Name");
while(buffer.length){
var head = buffer.slice(0, 4);
if(head.toString() != "FILE"){
console.log("ERROR!!!!");
process.exit(1);
}
var sizeHex = buffer.slice(4, 8);
var size = parseInt(sizeHex, 16);
var content = buffer.slice(8, size + 8);
var delimiter = buffer.slice(size + 8, size + 9);
if(delimiter != "@"){
console.log("wrong delimiter!!!");
process.exit(1);
}
writeStream.write(content);
buffer = buffer.slice(size + 9);
}
setTimeout(function(){
writeStream.end();
}, 2000);
});
});
server.listen(6000);
Ram Usage in system Monitor
Before Running Above Script : 1.6gb of 6 gb
After Running Above Script : 1.8 gb of 6gb
-
Have you tried unsetting pack after writing it to socket so gc can pick it up? Like:
delete pack; global.gc();– dkasipovic Commented Aug 22, 2018 at 9:12 - 1 @DamirKasipovic Tried it But Still same issue – Neeraj Walia Commented Aug 22, 2018 at 9:39
- What node.js version are you using? – m1ch4ls Commented Aug 22, 2018 at 11:32
- I am Using node v10.6.0 @m1ch4ls – Neeraj Walia Commented Aug 22, 2018 at 18:17
1 Answer
Reset to default 5The problem is you are not waiting for socket.write to finish... The callback in socket.write is there to signal that writing has finished and you can send another chunk.
Instead of manually writing to socket use pipe or pipeline to manage the streaming for you.
This is my take:
client.js
const net = require('net');
const fs = require('fs');
const { pipeline } = require('stream');
const socket = new net.Socket();
socket.connect(6000, '127.0.0.1');
socket.on('connect', function () {
const fileStream = fs.createReadStream('/dev/zero', { highWaterMark: 16384, end: 2 * 1024 * 1024 * 1024 }); // read 2GB of zeros, replace with real file
console.log('New file transfer');
pipeline(
fileStream,
socket,
(error) => {
if (error) { console.error(error) }
console.log('File transfer done');
}
);
});
server.js
const net = require('net');
const fs = require('fs');
const { pipeline } = require('stream');
const server = net.createServer(function (socket) {
const fileStream = fs.createWriteStream('/dev/null');
console.log('New file transfer');
pipeline(
socket,
fileStream,
(error) => {
if (error) { console.error(error) }
console.log('File transfer done');
}
)
});
server.listen(6000);
From what I have tested it never exceeds 100MB in RAM and overall the code behaves reasonably - so no gc and memory checks are necessary.
The code above is using pipeline function that is available only in latest Node.js 10 - in case you use older Node use pump package that works the same way.