javascript - HTTP request to an external API in Firebase Cloud Functions (Spark Tier) refused

Question

I am trying to call an external resource from the web and load the results into Dialogflow using the NodeJS Client V2 and Cloud Functions.

I have tried multiple combinations of this code, using promises, external functions, etc. No luck.

Test1

function myIntent(conv) {
            // console.log('conv ', conv)

            const config ={
                "headers": {'X-App-Token': dataSFAPITOken},
                "Content-Type": "application/json"
            }
            const url = "http://data.sfgov.org/resource/cuks-n6tp.json";
            return new Promise((resolve, reject) => {

                axios
                    .get(url)
                    .then(response => {
                        console.log(response);
                        conv.ask('I got your data back')
                        return resolve(response)
                    })
                    .catch(error => {
                        console.log(error);
                        conv.ask('Sorry! Something went wrong')
                        return reject(error)
                    });
            })

    }
 app.intent('My Intent', myIntent);  
 exports.myAgent = functions.https.onRequest(app);

Error

This is the error that I get on the Cloud Functions Dashboard, after I invoke myIntent .

{ Error: getaddrinfo ENOTFOUND data.sfgov.org data.sfgov.org:80
    at errnoException (dns.js:28:10)
    at GetAddrInfoReqWrap.onlookup [as oncomplete] (dns.js:76:26)
  code: 'ENOTFOUND',
  errno: 'ENOTFOUND',
  syscall: 'getaddrinfo',
  hostname: 'data.sfgov.org',
  host: 'data.sfgov.org',
  port: 80,
  config: 
   { adapter: [Function: httpAdapter],
     transformRequest: { '0': [Function: transformRequest] },
     transformResponse: { '0': [Function: transformResponse] },
     timeout: 0,
     xsrfCookieName: 'XSRF-TOKEN',
     xsrfHeaderName: 'X-XSRF-TOKEN',
     maxContentLength: -1,
     validateStatus: [Function: validateStatus],
     headers: 
      { Accept: 'application/json, text/plain, */*',
        'User-Agent': 'axios/0.18.0' },
     method: 'get',
     url: 'http://data.sfgov.org/resource/cuks-n6tp.json',
     data: undefined },
  request: 
   Writable {
     _writableState: 
      WritableState {
        objectMode: false,
        highWaterMark: 16384,
        needDrain: false,
        ending: false,
        ended: false,
        finished: false,
        decodeStrings: true,
        defaultEncoding: 'utf8',
        length: 0,
        writing: false,
        corked: 0,
        sync: true,
        bufferProcessing: false,
        onwrite: [Function],
        writecb: null,
        writelen: 0,
        bufferedRequest: null,
        lastBufferedRequest: null,
        pendingcb: 0,
        prefinished: false,
        errorEmitted: false,
        bufferedRequestCount: 0,
        corkedRequestsFree: [Object] },
     writable: true,
     domain: null,
     _events: 
      { response: [Function: handleResponse],
        error: [Function: handleRequestError] },
     _eventsCount: 2,
     _maxListeners: undefined,
     _options: 
      { protocol: 'http:',
        maxRedirects: 21,
        maxBodyLength: 10485760,
        path: '/resource/cuks-n6tp.json',
        method: 'get',
        headers: [Object],
        agent: undefined,
        auth: undefined,
        hostname: 'data.sfgov.org',
        port: null,
        nativeProtocols: [Object],
        pathname: '/resource/cuks-n6tp.json' },
     _redirectCount: 0,
     _requestBodyLength: 0,
     _requestBodyBuffers: [],
     _onNativeResponse: [Function],
     _currentRequest: 
      ClientRequest {
        domain: null,
        _events: [Object],
        _eventsCount: 6,
        _maxListeners: undefined,
        output: [],
        outputEncodings: [],
        outputCallbacks: [],
        outputSize: 0,
        writable: true,
        _last: true,
        upgrading: false,
        chunkedEncoding: false,
        shouldKeepAlive: false,
        useChunkedEncodingByDefault: false,
        sendDate: false,
        _removedHeader: {},
        _contentLength: 0,
        _hasBody: true,
        _trailer: '',
        finished: true,
        _headerSent: true,
        socket: [Object],
        connection: [Object],
        _header: 'GET /resource/cuks-n6tp.json HTTP/1.1
Accept: application/json, text/plain, */*
User-Agent: axios/0.18.0
Host: data.sfgov.org
Connection: close

',
        _headers: [Object],
        _headerNames: [Object],
        _onPendingData: null,
        agent: [Object],
        socketPath: undefined,
        timeout: undefined,
        method: 'GET',
        path: '/resource/cuks-n6tp.json',
        _ended: false,
        _redirectable: [Circular],
        parser: null },
     _currentUrl: 'http://data.sfgov.org/resource/cuks-n6tp.json' },
  response: undefined }  

Test2

const url = "data.sfgov.org";
var options = {
    protocol:'http:',
    host: url,
    port : 8080,
    path:'/resource/cuks-n6tp',
    headers: {
        'X-App-Token': dataSFAPITOken,
        "Content-Type": "application/json"
    }
};
http.get(options, (http_res) => {
    // initialize the container for our data
    var data = "";

    // this event fires many times, each time collecting another piece of the response
    http_res.on("data", (chunk) =>{
        // append this chunk to our growing `data` var
        data += chunk;
    });

    // this event fires *one* time, after all the `data` events/chunks have been gathered
    http_res.on("end", () => {
        // you can use res.send instead of console.log to output via express
        console.log(data);
    });
});

Here is the link to the official documentation HERE.

The crazy thing about this is that below is a similar implementation on the browser and it works. As a matter of fact, I don't need an API key to query the data. I can simply copy/paste the link.

$.ajax({
    url: "https://data.sfgov.org/resource/cuks-n6tp.json",
    type: "GET",
    data: {
      "$limit" : 50,
      "$$app_token" : "APPTOKEN"
    }
}).done(function(data) {
  // alert("Retrieved " + data.length + " records from the dataset!");
  console.log(data);
  document.getElementById('data').innerHTML = JSON.stringify(data, null,2)
});

But for some reasons, I can't make it work with Dialogflow NodeJS Client V2. I am pretty sure in V1, I can make it work.

My migration to V2 is a little painful. Please help.

Thanks.

Answer 1

Waitting for answers