Want to win a PS4? Go Premium and enter to win our High-Tech Treats giveaway. Enter to Win

x
?
Solved

gzip in express of nodejs

Posted on 2016-07-21
12
Medium Priority
?
977 Views
Last Modified: 2016-10-01
Hello Experts.

How to make gzip working?
Below is a code, but it fails:

   var express = require("express"),
   server = express(),
   ...
    server.use(bodyParser.json());
    server.use(bodyParser.urlencoded({ extended: true }));
    server.use( compression() );
    //breaks: server.use( express.compress( {threshold: 0} ) );

    var topDir = __dirname + "/../..";
    server.use(express.static( topDir )); // location is relative to this file, so be carful
    server.all( '/ws/*', function( req, res ) {
        var handleResponse = function (error, resp, body) {
        ...


    the gzip data comes through /ws/..., but from "express", it comes to the web browser as raw gzip, even
    if browser gets, Content-Encoding:gzip

   But, when go to /ws/* directly from the browser: gzip is handled correctly


   "breaks" means the following mesage:
Error: Most middleware (like compress) is no longer bundled with Express and must be installed separately. Please see https://github.com/senchalabs/connect#middleware.
    at Function.Object.defineProperty.get (/var/www/proxy/node_modules/express/lib/express.js:99:13)
    at Object.<anonymous> (/var/www/proxy/rhtml/rms/rbserver.js:38:25)
    ....


  Thank you.
0
Comment
Question by:Bitlab
[X]
Welcome to Experts Exchange

Add your voice to the tech community where 5M+ people just like you are talking about what matters.

  • Help others & share knowledge
  • Earn cash & points
  • Learn & ask questions
  • 8
  • 4
12 Comments
 
LVL 27

Expert Comment

by:BigRat
ID: 41724490
Where is the following statement :-

var compression = require('compression')

and have you installed it as according to https://www.npmjs.com/package/compression ?
0
 
LVL 2

Author Comment

by:Bitlab
ID: 41724959
Thank you for reply.

The express server runs by script:
    #!/bin/bash -e
    nodejs rhtml/rms/rbserver.js

Response Header got from express ( aka: web-browser <-- express <-- real-web-server ) is:
    Access-Control-Allow-Headers:*
    Access-Control-Allow-Origin:*
    Connection:keep-alive
    Content-Encoding:gzip
    Content-Type:application/json
    Date:Fri, 22 Jul 2016 15:38:36 GMT
    Transfer-Encoding:chunked
    Vary:Accept-Encoding
    X-Powered-By:Express
But response itself appears in browser as raw gzip-content:
    ��=�W��]is9��+��f�&՟F�նdj=���鍍ؒX2�<J�Cyb��&�D!d......
Response size = 8.9 KB


The code in question is in file rbserver.js: the code =
    var express = require("express"),
       server = express(),
       ...
    server.use(bodyParser.json());
    server.use(bodyParser.urlencoded({ extended: true }));
    server.use( compression() );

$ npm install compression
    npm WARN package.json myproxy@0.0.1 No README data
    npm http GET https://registry.npmjs.org/compression
    npm http 304 https://registry.npmjs.org/compression
    npm http GET https://registry.npmjs.org/bytes/2.3.0
    npm http GET https://registry.npmjs.org/compressible
    npm http GET https://registry.npmjs.org/accepts
    npm http GET https://registry.npmjs.org/debug
    npm http GET https://registry.npmjs.org/vary
    npm http GET https://registry.npmjs.org/on-headers
    npm http 304 https://registry.npmjs.org/compressible
    npm http 200 https://registry.npmjs.org/debug
    npm http 200 https://registry.npmjs.org/accepts
    npm http 200 https://registry.npmjs.org/vary
    npm http 304 https://registry.npmjs.org/on-headers
    npm http 304 https://registry.npmjs.org/bytes/2.3.0
    npm http GET https://registry.npmjs.org/mime-db
    npm http GET https://registry.npmjs.org/mime-types
    npm http GET https://registry.npmjs.org/negotiator/0.6.1
    npm http GET https://registry.npmjs.org/ms/0.7.1
    npm http 304 https://registry.npmjs.org/mime-db
    npm http 304 https://registry.npmjs.org/negotiator/0.6.1
    npm http 200 https://registry.npmjs.org/mime-types
    npm http 200 https://registry.npmjs.org/ms/0.7.1
    npm http GET https://registry.npmjs.org/ms/-/ms-0.7.1.tgz
    npm http 200 https://registry.npmjs.org/ms/-/ms-0.7.1.tgz
    compression@1.6.2 node_modules/compression
    ├── on-headers@1.0.1
    ├── vary@1.1.0
    ├── bytes@2.3.0
    ├── compressible@2.0.8 (mime-db@1.23.0)
    ├── debug@2.2.0 (ms@0.7.1)
    └── accepts@1.3.3 (negotiator@0.6.1, mime-types@2.1.11)
0
 
LVL 2

Author Comment

by:Bitlab
ID: 41724970
Here is more details about the code from the beginning:

    var express = require("express"),
        http = require("http"),
        https = require("https"),
        port = (process.env.PORT || 8080),
        httpsPort = 443,
        server = express(),
        httpProxy = require('http-proxy'),
        requestMod = require('request'),
        bodyParser = require('body-parser'),
        compression = require('compression');
        var proxyUrl = {
            office: 'http://10.10.10.10', //aka
        };

        var headers = {
            "Content-Type": "application/json",
            "Access-Control-Allow-Origin": "*",
            "Access-Control-Allow-Headers": "*"
        };

        var urlRootToUse = proxyUrl.office;
       
        //.possibly irrelevant and useless for rms:
        server.use(bodyParser.json()); // for parsing application/json

        server.use(bodyParser.urlencoded({ extended: true })); // for parsing application/x-www-form-urlencoded
        server.use( compression() );    //npm i express --save; npm i compression --save
        //server.use( express.compress( {threshold: 0} ) );

        var topDir = __dirname + "/../..";
        server.use(express.static( topDir )); // location is relative to this file, so be carful
        server.all( '/ws/*', function( req, res ) {
            var handleResponse = function (error, resp, body) {
            .....
0
Independent Software Vendors: We Want Your Opinion

We value your feedback.

Take our survey and automatically be enter to win anyone of the following:
Yeti Cooler, Amazon eGift Card, and Movie eGift Card!

 
LVL 27

Expert Comment

by:BigRat
ID: 41724980
I suspect that the browser wants a "Content-Transfer-Encoding" rather than a "Content-Encoding". This has always been a mute point with regard to the protocol. Have you set the type of encoding anywhere (in the ... part perhaps)?
0
 
LVL 2

Author Comment

by:Bitlab
ID: 41726802
Thank you. I added this header. But no dice yet. Here is what I have at the moment:
Must I manually gzip the body in res.end(e); ?


Apache sends all headers:
    Accept-Ranges:bytes
    Cache-Control:no-cache
    Content-Encoding:gzip
    Content-Length:612
    Content-Transfer-Encoding:gzip
    Content-Type:application/json
    Date:Sun, 24 Jul 2016 21:44:44 GMT
    Expires:Thu, Jan 1 1970 00:00:00 GMT
    Server:Apache/2.2.15 (CentOS)


express sends to browser nearly all headers:
    Access-Control-Allow-Headers:*
    Access-Control-Allow-Origin:*
    Connection:keep-alive
    Content-Encoding:gzip
    Content-Type:application/json
    Date:Sun, 24 Jul 2016 21:35:37 GMT
    Transfer-Encoding:chunked
    Vary:Accept-Encoding
    X-Powered-By:Express



but express does not console.log full headers: why?
must I manually gzip the body in res.end(e); ?

    headers= { 'Content-Type': 'application/json',
      'Access-Control-Allow-Origin': '*',
      'Access-Control-Allow-Headers': '*' }

    where code is:
    server.all( '/ws/*', function( req, res ) {

        var handleResponse = function (error, resp, body) {

            try {
                if(resp && resp.statusCode){
                    res.writeHead(resp.statusCode, headers);
console.log( 'headers=', headers );
                } else {
                    res.writeHead(400, headers);
console.log( '400, headers=', headers );
                }
                if(body || !error){
                    res.end(body);
                }
                if(error){
                    console.log('error', error);
                    res.end(error);
                }
            } catch(e){
                console.log(e);
                res.end(e);
            } finally {
                res.end();
            }

        };

        try {
            var urlToUse = urlRootToUse + req.url;
            if (req.method === 'GET'){
                requestMod.get(urlToUse, handleResponse);
            } else if(req.method === "POST"){
                //c onsole.log( 'req.body=', req.body );
                requestMod.post({url: urlToUse, form: req.body }, handleResponse);
            }
        } catch(e){
            console.log(e);
            res.send(e);
        }
    });

Thank you.
0
 
LVL 2

Author Comment

by:Bitlab
ID: 41726808
I meant:
"Must I manually gzip the body in res.end(body); ?"
0
 
LVL 27

Expert Comment

by:BigRat
ID: 41727880
Hmm. I don't like the content-encoding:gzip since the content isn't encoded as such. Who added this heaer to Apache's response? I don't remember it being added automatically. And what "object" is requestMod?
0
 
LVL 2

Author Comment

by:Bitlab
ID: 41814183
requestMod = require('request'),
This line is a part of the dev-server.js
which is run as
node dev-server.js

I already provided this line in my comments above starting with the words:
"Here is more details about the code from the beginning:"

Anyway, there is a full JavaScript file attached.
dev-server.js
0
 
LVL 2

Author Comment

by:Bitlab
ID: 41814189
In regards to this comment "....I don't like the content-encoding:gzip since the content isn't encoded as such. Who added this heaer to Apache's response?..."

I added.
Everything works fine when I don't use nodejs/express. I mean when  browser talks directly to Apache:
Apache sends compressed file and browser nicely understands it.
But nodejs/express fails.

How did I add it? Here is a sample. This sample is in Python, but it is clear:

      #https://en.wikipedia.org/wiki/HTTP_compression
      #npm install compression
      start_response( '200 OK', [
            ('Content-Type', 'application/json'),
            ('Accept-Ranges', 'bytes' ),
            ('Content-Encoding', 'gzip' ),
            #('Content-Transfer-Encoding', 'gzip' ), #This seems only for e-mail, seems did not hurt, so far, but some say it can:
            #http://stackoverflow.com/questions/7285372/is-content-transfer-encoding-an-http-header
            ('Content-Length', str(len( result_str ))),
            ('Cache-Control', 'no-cache'),
            ('Expires', 'Thu, Jan 1 1970 00:00:00 GMT'),
            ])

Thank you.
0
 
LVL 27

Accepted Solution

by:
BigRat earned 2000 total points
ID: 41814736
I often can't see the wood for the trees. I suspect that when using node the response contains a compressed body but without a corresponding header. Looking at the Javascript you are passing the request onwards and the reply is being sent back using a fixed header :-

if(resp && resp.statusCode){
                    res.writeHead(resp.statusCode, headers);
                } else {
                    res.writeHead(400, headers);
                }

instead of the header returned from above. In order to check this I suggest starting the node server and using siomething to look at the headers returned. A simple telnet would work but you could use something like fiddler.
0
 
LVL 2

Author Comment

by:Bitlab
ID: 41817162
Thank you. It looks you are right, I must pass over the headers that came from Apache to web-browser Sorry, I am slow in response, hopefully will find time to complete this research.
0
 
LVL 2

Author Closing Comment

by:Bitlab
ID: 41825042
Thank you. Since I have to postpone this project, I will rate it right now and may be return to it later.
0

Featured Post

Plesk WordPress Toolkit

Plesk's WordPress Toolkit allows server administrators, resellers and customers to manage their WordPress instances, enabling a variety of development workflows for WordPress admins of all skill levels, from beginners to pros.

See why 2/3 of Plesk servers use it.

Question has a verified solution.

If you are experiencing a similar issue, please ask a related question

When crafting your “Why Us” page, there are a plethora of pitfalls to avoid. Follow these five tips, and you’ll be well on your way to creating an effective page.
Q&A with Course Creator, Mark Lassoff, on the importance of HTML5 in the career of a modern-day developer.
Explain concepts important to validation of email addresses with regular expressions. Applies to most languages/tools that uses regular expressions. Consider email address RFCs: Look at HTML5 form input element (with type=email) regex pattern: T…
This tutorial demonstrates how to identify and create boundary or building outlines in Google Maps. In this example, I outline the boundaries of an enclosed skatepark within a community park.  Login to your Google Account, then  Google for "Google M…
Suggested Courses

604 members asked questions and received personalized solutions in the past 7 days.

Join the community of 500,000 technology professionals and ask your questions.

Join & Ask a Question