Unknown macro: {style}

.ia-secondary-container

Unknown macro: { margin}
Unknown macro: {div}
Kaa releases
Unknown macro: {div}
Shortcuts

Page tree
Skip to end of metadata
Go to start of metadata

The Kafka log appender is responsible for transferring logs from the Operations server to the Apache Kafka service. The logs are stored in the specified topic.

Creating Kafka log appender in Admin UI

The easiest way to create a Kafka log appender for your application is by using Admin UI.

Creating Kafka log appender with REST API

It is also possible to create a Kafka log appender for your application by using REST API. The following example illustrates how to provision the Kafka log appender for the Cell Monitor demo application available in Kaa Sandbox.

Configuration

The Kafka log appender configuration should match the following Avro schema.

{
    "namespace":"org.kaaproject.kaa.server.appenders.kafka.config.gen",
    "type":"record",
    "name":"KafkaConfig",
    "fields":[
        {
            "name":"kafkaServers",
            "displayName":"Kafka bootstrap servers",
            "minRowCount":1,
            "type":{
                "type":"array",
                "items":{
                    "namespace":"org.kaaproject.kaa.server.appenders.kafka.config.gen",
                    "type":"record",
                    "name":"KafkaServer",
                    "fields":[
                        {
                            "name":"host",
                            "displayName":"Host",
                            "weight":0.75,
                            "by_default":"localhost",
                            "type":"string"
                        },
                        {
                            "name":"port",
                            "displayName":"Port",
                            "weight":0.25,
                            "by_default":9092,
                            "type":"int"
                        }
                    ]
                }
            }
        },
        {
            "name":"topic",
            "displayName":"Topic name",
            "by_default":"kaa",
            "type":"string"
        },
        {
            "name":"useDefaultPartitioner",
            "displayName":"Use default partitioner",
            "by_default":true,
            "type":"boolean"
        },
        {
            "name":"partitionCount",
            "displayName":"Partition count",
            "by_default":1,
            "type":"int"
        },
        {
            "namespace":"org.kaaproject.kaa.server.appenders.kafka.config.gen",
            "name":"kafkaKeyType",
            "by_default":"NULL",
            "optional":true,
            "displayName":"Key type",
            "type":[
                {
                    "name":"KafkaKeyType",
                    "type":"enum",
                    "symbols":[
                        "ENDPOINTHASHKEY",
                        "HASH",
                        "UUID",
                        "NULL"
                    ]
                },
                "null"
            ]
        },
        {
            "name":"executorThreadPoolSize",
            "displayName":"Executor thread pool size",
            "by_default":1,
            "type":"int"
        },
        {
            "name":"bufferMemorySize",
            "displayName":"Buffer memory size",
            "by_default":33554432,
            "type":"long"
        },  
        {
            "namespace":"org.kaaproject.kaa.server.appenders.kafka.config.gen",
            "name":"kafkaCompression",
            "by_default":"NONE",
            "optional":true,
            "displayName":"Compression type",
            "type":[
                {
                    "name":"KafkaCompression",
                    "type":"enum",
                    "symbols":[
                        "NONE",
                        "SNAPPY",
                        "GZIP"
                    ]
                },
                "null"
            ]
        },
		{
            "namespace":"org.kaaproject.kaa.server.appenders.kafka.config.gen",
            "name":"kafkaAcknowledgement",
            "by_default":"ONE",
            "optional":true,
            "displayName":"Acknowledgement type",
            "type":[
                {
                    "name":"KafkaAcknowledgement",
                    "type":"enum",
                    "symbols":[
						"ALL",
                        "ZERO",
						"ONE",
						"TWO"
                    ]
                },
                "null"
            ]
        },
        {
            "name":"retries",
            "displayName":"Retries",
            "by_default":0,
            "type":"int"
        }                   
    ]
}

The following configuration taken from the Cell Monitor demo matches the previous schema.

{  
    "kafkaServers":[  
        {  
            "host":"localhost",
            "port":9092
        }
    ],
    "topic":"kaa",
	"useDefaultPartitioner":{
		"boolean":true
	},
    "partitionCount":{
		"int":1
	},
	"kafkaKeyType":{
        "org.kaaproject.kaa.server.appenders.kafka.config.gen.KafkaKeyType":"NULL"
    },
    "executorThreadPoolSize":{
		"int":1
	},
	"bufferMemorySize":{
		"long":33554432
	},
	"kafkaCompression":{
        "org.kaaproject.kaa.server.appenders.kafka.config.gen.KafkaCompression":"NONE"
    },
	"kafkaAcknowledgement":{
        "org.kaaproject.kaa.server.appenders.kafka.config.gen.KafkaAcknowledgement":"ONE"
    },
    "retries":{
		"int":0
	}
}


Administration

The following REST API call example illustrates how to create a new Kafka log appender.

curl -v -S -u devuser:devuser123 -X POST -H 'Content-Type: application/json' -d'{"pluginClassName": "org.kaaproject.kaa.server.appenders.kafka.appender.KafkaLogAppender", "applicationId": 119, "applicationToken": "91786338058670361194", "jsonConfiguration": "{\"kafkaServers\":[{\"host\":\"localhost\",\"port\":9092}],\"topic\":\"kaa\",\"useDefaultPartitioner\":{\"boolean\":true},\"partitionCount\":{\"int\":1},\"kafkaKeyType\":{\"org.kaaproject.kaa.server.appenders.kafka.config.gen.KafkaKeyType\":\"NULL\"},\"executorThreadPoolSize\":{\"int\":1},\"bufferMemorySize\":{\"long\":33554432},\"kafkaCompression\":{\"org.kaaproject.kaa.server.appenders.kafka.config.gen.KafkaCompression\":\"NONE\"},\"kafkaAcknowledgement\":{\"org.kaaproject.kaa.server.appenders.kafka.config.gen.KafkaAcknowledgement\":\"ONE\"},\"retries\":{\"int\":0}}", "description": "New sample Kafka log appender", "headerStructure": [ "KEYHASH","TIMESTAMP" ], "name": "New Kafka appender", "maxLogSchemaVersion": 2147483647, "minLogSchemaVersion": 1, "tenantId": "70"}' "http://localhost:8080/kaaAdmin/rest/api/logAppender" | python -mjson.tool
 Example result
{
    "appenderClassName": "org.kaaproject.kaa.server.appenders.kafka.appender.KafkaLogAppender",
    "applicationId": 70,
    "applicationToken": "946558468095768",
    "configuration": "{\"kafkaServers\":[{\"host\":\"localhost\",\"port\":9092}],\"topic\":\"kaa\",\"useDefaultPartitioner\":{\"boolean\":true},\"partitionCount\":{\"int\":1},\"kafkaKeyType\":{\"org.kaaproject.kaa.server.appenders.kafka.config.gen.KafkaKeyType\":\"NULL\"},\"executorThreadPoolSize\":{\"int\":1},\"bufferMemorySize\":{\"long\":33554432},\"kafkaCompression\":{\"org.kaaproject.kaa.server.appenders.kafka.config.gen.KafkaCompression\":\"NONE\"},\"kafkaAcknowledgement\":{\"org.kaaproject.kaa.server.appenders.kafka.config.gen.KafkaAcknowledgement\":\"ONE\"},\"retries\":{\"int\":0}}",
    "createdTime":1417105170741,
    "createdUsername":"devuser",
    "description": "New sample Kafka log appender",
    "headerStructure": [
        "KEYHASH",
        "TIMESTAMP"
    ],
    "id":"164",
    "name": "New Kafka appender",
    "maxLogSchemaVersion": 2147483647,
    "minLogSchemaVersion": 1,
    "tenantId": "10"
    "typeName":"Kafka"
}

Copyright © 2014-2015, CyberVision, Inc.

  • No labels