INTEGRATIONS

AWS Kinesis with Lambda function

logs from kinesis into Coralogix

Coralogix provides a predefined Lambda function to forward your Kinesis stream straight to Coralogix.

Setup

  1. Create an “author from scratch” Node.js 8.10 runtime lambda with the following permissions(Create execution role tutorial):
  2. At “Code entry type” choose “Edit code inline” and paste the following function:
'use strict';

const AWS = require('aws-sdk');
const https = require('https');
const assert = require('assert');

assert(process.env.private_key, 'No private key')
const appName = process.env.app_name ? process.env.app_name : 'NO_APPLICATION';
const subName = process.env.sub_name ? process.env.sub_name : 'NO_SUBSYSTEM';

exports.handler = (event, context, callback) => {

    function parseEvent(streamEventRecord) {
        let streamEventData = new Buffer(streamEventRecord.kinesis.data, 'base64').toString('ascii');
        return {
            "timestamp": streamEventRecord.kinesis.approximateArrivalTimestamp * 1000,
            "severity": getSeverityLevel(streamEventData),
            "text": streamEventData
        };
    }

    function postEventsToCoralogix(parsedEvents) {
        try {
            var options = {
                hostname: 'api.coralogix.com',
                port: 443,
                path: '/api/v1/logs',
                method: 'POST',
                headers: {
                    'Content-Type': 'application/json',
                }
            };

            let retries = 3;
            let timeoutMs = 10000;
            let retryNum = 0;
            let sendRequest = function sendRequest() {
                let req = https.request(options, function (res) {
                    console.log('Status: ' + res.statusCode);
                    console.log('Headers: ' + JSON.stringify(res.headers));
                    res.setEncoding('utf8');
                    res.on('data', function (body) {
                        console.log('Body: ' + body);
                    });
                });

                req.setTimeout(timeoutMs, () => {
                    req.abort();
                    if (retryNum++ < retries) {
                        console.log('problem with request: timeout reached. retrying ' + retryNum + '/' + retries);
                        sendRequest();
                    } else {
                        console.log('problem with request: timeout reached. failed all retries.');
                    }
                });

                req.on('error', function (e) {
                    console.log('problem with request: ' + e.message);
                });

                req.write(JSON.stringify(parsedEvents));
                req.end();
            }

            sendRequest();
        } catch (ex) {
            console.log(ex.message);
            callback(ex.message);
        }
    }

    function getSeverityLevel(message) {
        var severity = 3;

        if(message.includes('debug'))
            severity = 1
        if(message.includes('verbose'))
            severity = 2
        if(message.includes('info'))
            severity = 3
        if(message.includes('warn') || message.includes('warning'))
            severity = 4
        if(message.includes('error'))
            severity = 5
        if(message.includes('critical') || message.includes('panic'))
            severity = 6

        return severity;
    }

    postEventsToCoralogix({
        "privateKey": process.env.private_key,
        "applicationName": appName,
        "subsystemName": subName,
        "logEntries": event.Records.map((eventRecord) => parseEvent(eventRecord))
    });
};
  1. Add the mandatory environment variables:
    • Private Key – A unique ID which represents your company, this Id will be sent to your mail once you register to Coralogix.
    • Application Name – Used to separate your environment, e.g. SuperApp-test/SuperApp-prod.
    • SubSystem Name – Your application probably has multiple subsystems, for example, Backend servers, Middleware, Frontend servers etc.

    Amazon kinesis integration coralogix environment variables

  2. Go to Add triggers and add Kinesis:
    amazon kinesis integration coralogix designer
  3. Configure the trigger, select the desired “Kinesis stream” and “Consumer”, change “Batch size” equals to 10:
    amazon kinesis integration coralogix configure triggers
  4. Increase Memory to 1024MB and Timeout to 1 min.
    amazon kinesis integration coralogix basic settings
  5. Click “Save”.

Let's talk about how Coralogix can help you better understand your logs