The Kaa logging subsystem is designed to collect records (logs) of pre-configured structure on endpoints, periodically deliver these records from endpoints to Operation servers, and either persist them on the server for further processing or submit to immediate stream analysis. The log structure in Kaa is determined for each application by a configurable log schema. On the Operation server side, there are log appenders which are responsible for writing logs received by the Operations server into the specific storage. A Kaa tenant administrator can define only one log appender per application.
Please view logging system design reference for more background information.
From this guide you will learn how to use the Kaa logging subsystem for data collection.
The following diagram illustrates basic entities and data flows in scope of the log management:
This section illustrates how to configure a log schema.
The default log schema installed for Kaa applications is empty. You can configure your own log schema using the Admin UI or REST API. For the purpose of this guide, we will use schema that is very close to the common log structure: the log level, tag and message.
{ "type":"record", "name":"LogData", "namespace":"org.kaaproject.kaa.schema.sample.logging", "fields":[ { "name":"level", "type":{ "type":"enum", "name":"Level", "symbols":[ "DEBUG", "ERROR", "FATAL", "INFO", "TRACE", "WARN" ] } }, { "name":"tag", "type":"string" }, { "name":"message", "type":"string" } ] }
Kaa provides default implementations of log appenders that store logs in Hadoop, Cassandra, MongoDB or a local file system (FS). It is possible to develop and integrate custom log appenders.
The logging subsystem API varies depending on the target SDK platform. However, the general approach is the same.
To transfer logs to the Kaa Operation server, the Kaa client application should use the following code.
// Configure the log delivery listener kaaClient.setLogDeliveryListener(new LogDeliveryListener() { @Override public void onLogDeliverySuccess(BucketInfo bucketInfo) { /* Called on success */ } @Override public void onLogDeliveryFailure(BucketInfo bucketInfo) { /* Called on failure */ } @Override public void onLogDeliveryTimeout(BucketInfo bucketInfo) { /* Called on timeout */ } }); // Create a log entity according to the (org.kaaproject.sample.LogData) sample schema above LogData logRecord = new LogData(Level.INFO, "tag", "message"); // Push the record to the collector RecordFuture logDeliveryStatus = kaaClient.addLogRecord(logRecord); // Get log delivery information RecordInfo logDeliveryReport = logDeliveryStatus.get();
#include <iostream> #include <exception> include <kaa/Kaa.hpp> using namespace kaa; // Create an endpoint instance auto kaaClient = Kaa::newClient(); // Start an endpoint kaaClient->start(); // Create a log entity (according to the org.kaaproject.sample.LogData sample schema above) KaaUserLogRecord logRecord; logRecord.level = Level::INFO; logRecord.tag = "tag"; logRecord.message = "message"; // Push the record to the collector auto recordDeliveryCallback = kaaClient->addLogRecord(logRecord); try { auto recordInfo = recordDeliveryCallback.get(); auto bucketInfo = recordInfo.getBucketInfo(); std::cout << "Received log record delivery info. Bucket Id [" << bucketInfo.getBucketId() << "]. " << "Record delivery time [" << recordInfo.getRecordDeliveryTimeMs() << " ms]." << std::endl; } catch (std::exception& e) { std::cout << "Exception was caught while waiting for callback result: " << e.what() << std::endl; }
#include <stdint.h> #include <kaa/kaa_logging.h> #include <kaa/gen/kaa_logging_gen.h> #include <kaa/platform/kaa_client.h> kaa_client_t *kaa_client = /* ... */; void *log_storage_context = NULL; void *log_upload_strategy_context = NULL; /* Optional context that passed to log delivery callbacks */ void *log_delivery_context = NULL; /* Assume Kaa SDK is already initialized */ /* Set of routines that handles log delivery events */ static void success_log_delivery_callback(void *context, const kaa_log_bucket_info_t *bucket) { /* ... */ } static void failed_log_delivery_callback(void *context, const kaa_log_bucket_info_t *bucket) { /* ... */ } static void timeout_log_delivery_callback(void *context, const kaa_log_bucket_info_t *bucket) { /* ... */ } /* Log delivery listener callbacks. Each callback called whenever something happen with a log bucket. */ kaa_log_delivery_listener_t log_listener = { .on_success = success_log_delivery_callback, /* Called if log delivered successfully */ .on_failed = failed_log_delivery_callback, /* Called if delivery failed */ .on_timeout = timeout_log_delivery_callback, /* Called if timeout occurs */ .ctx = log_delivery_context, /* Optional context */ }; /* The internal memory log storage distributed with Kaa SDK */ kaa_error_t error_code = ext_unlimited_log_storage_create(&log_storage_context, kaa_client_get_context(kaa_client)->logger); /* Check error code */ /* Specify log bucket size constraints */ kaa_log_bucket_constraints_t bucket_sizes = { .max_bucket_size = 512, /* Bucket size in bytes */ .max_bucket_log_count = 5, /* Maximum log count in one bucket */ }; /* Initialize the log storage and strategy (by default it is not set) */ error_code = kaa_logging_init(kaa_client_get_context(kaa_client)->log_collector , log_storage_context , log_upload_strategy_context , &bucket_sizes); /* Check error code */ /* Add listeners to a log collector */ kaa_logging_set_listeners(kaa_client_get_context(kaa_client)->log_collector, &log_listener); /* Create and add a log record */ kaa_user_log_record_t *log_record = kaa_logging_log_data_create(); log_record->level = ENUM_LEVEL_KAA_TRACE; log_record->tag = kaa_string_copy_create("SOME_TAG"); log_record->message = kaa_string_copy_create("SOME_MESSAGE"); /* Log information. Populated when log is added via kaa_logging_add_record() */ kaa_log_record_info_t log_info; /* Add log record */ error_code = kaa_logging_add_record(kaa_client_get_context(kaa_client)->log_collector, log_record, &log_info); /* Check error code */ log_record->destroy(log_record);
// Create a log entity (according to the org.kaaproject.sample.LogData sample schema above) LogData *logRecord = [[LogData alloc] initWithLevel:LEVEL_INFO tag:@"tag" message:@"message"]; // Push the record to the collector BucketRunner *runner = [kaaClient addLogRecord:logRecord]; // Add callback for log delivery @try { [[[NSOperationQueue alloc] init] addOperationWithBlock:^{ BucketInfo *bucketInfo = [runner getValue]; NSLog(@"Received log record delivery info. Bucket Id [%d]. Record delivery time [%f ms]", bucketInfo.bucketId, bucketInfo.bucketDeliveryDuration); }]; } @catch (NSException *exception) { NSLog(@"Exception was caught while waiting for callback"); }
By default, the Kaa SDK uses an in-memory log storage. Normally, this storage does not persist data when the client is restarted. If this is a concern, Java/Objective-C/C++ SDKs provide a persistent log storage based on SQLite database.
Here is an example how to use SQLite log storage for Java/Objective-C/C++ SDKs and a custom implementation for C SDK:
// Default SQLite database name String databaseName = "kaa_logs"; // Default maximum bucket size in bytes int maxBucketSize = 16 * 1024; // Default maximum amount of log records in a bucket int maxRecordCount = 256; // Setting SQLite log storage implementation kaaClient.setLogStorage(new DesktopSQLiteDBLogStorage(databaseName, maxBucketSize, maxRecordCount));
// Setting SQLite log storage implementation kaaClient.setLogStorage(new AndroidSQLiteDBLogStorage(/*Android context*/, "kaa_logs"/* default value */, 16 * 1024/* default value */, 256/* default value */)));
#include <memory> #include <kaa/log/SQLiteDBLogStorage.hpp> using namespace kaa; ... auto persistentStorage = std::make_shared<SQLiteDBLogStorage>(kaaClient->getKaaClientContext()); // Setting SQLite log storage implementation kaaClient->setStorage(persistentStorage);
#include <stdint.h> #include <kaa/kaa_logging.h> #include <kaa/platform/kaa_client.h> kaa_client_t *kaa_client = NULL /* ... */; /* * Log storage described in "kaa/platform/ext_log_storage.h" */ void *persistent_log_storage_context = NULL; /* * Log upload strategy described in "kaa/platform/ext_log_upload_strategy.h" */ void *persistent_log_upload_strategy_context = NULL; /* * Assume Kaa SDK is already initialized. * Create persistent_log_storage_context and persistent_log_upload_strategy_context instances. */ /* Specify log bucket size constraints */ kaa_log_bucket_constraints_t bucket_sizes = { .max_bucket_size = 512, /* Bucket size in bytes */ .max_bucket_log_count = 5, /* Maximum log count in one bucket */ }; /* Initialize the log storage and strategy (by default it is not set) */ kaa_error_t error_code; error_code = kaa_logging_init(kaa_client_get_context(kaa_client)->log_collector , persistent_log_storage_context , persistent_log_upload_strategy_context , &bucket_sizes); /* Check error code */
// Default maximum bucket size in bytes int maxBucketSize = 16 * 1024; // Default maximum amount of log records in a bucket int maxRecordCount = 256; // Setting log storage implementation. [kaaClient setLogStorage:[[SQLiteLogStorage alloc] initWithBucketSize:maxBucketSize bucketRecordCount:maxRecordCount]];
A log upload strategy determines under what conditions Kaa endpoints must send log data to the server. Kaa provides several built-in strategies, namely:
Periodic strategy to upload logs after at least the given amount of time passes since the last upload:
// Configure the strategy to upload no less than a hour worth of logs kaaClient.setLogUploadStrategy(new PeriodicLogUploadStrategy(60, TimeUnit.MINUTES));
// Configure the strategy to upload logs each 60 seconds kaaClient->setLogUploadStrategy(std::make_shared<kaa::PeriodicLogUploadStrategy>(60, kaaClient->getKaaClientContext()));
#include <stdint.h> #include <kaa/kaa_logging.h> #include <kaa/platform/kaa_client.h> #include <kaa/platform-impl/common/ext_log_upload_strategies.h> kaa_client_t *kaa_client = NULL; /* * The log storage. */ void *log_storage_context = NULL; /* * Log upload strategy. */ void *log_upload_strategy_context = NULL; /* * Assume Kaa SDK and log storage are already initialized. */ kaa_log_bucket_constraints_t bucket_sizes = { /* Specify bucket size constraints */ }; /* Create a strategy based on timeout. */ kaa_error_t error_code = ext_log_upload_strategy_create(kaa_client_get_context(kaa_client), &log_upload_strategy_context, KAA_LOG_UPLOAD_BY_TIMEOUT_STRATEGY); /* Check error code */ /* Strategy will upload logs every 5 seconds. */ error_code = ext_log_upload_strategy_set_upload_timeout(log_upload_strategy_context, 5); /* Check error code */ /* Initialize the log storage and strategy (by default it is not set) */ error_code = kaa_logging_init(kaa_client_get_context(kaa_client)->log_collector , log_storage_contex , log_upload_strategy_context , &bucket_sizes);
// Create log upload strategy, which will upload logs every 20 seconds PeriodicLogUploadStrategy *uploadStrategy = [[PeriodicLogUploadStrategy alloc] initWithTimeLimit:20 timeUnit:TIME_UNIT_SECONDS]; // Configure client to use our newly created strategy [kaaClient setLogUploadStrategy:uploadStrategy];
NOTE: The decision of whether to upload the logs collected is taken each time a new log record is added. That being said, the next log record added after the time specified passes will trigger a log upload.
Log count strategy to upload logs after the threshold of log records generated is reached:
// Configure the strategy to upload logs every fifth log record added kaaClient.setLogUploadStrategy(new RecordCountLogUploadStrategy(5)); // Configure the strategy to upload logs immediately kaaClient.setLogUploadStrategy(new RecordCountLogUploadStrategy(1));
// Configure the strategy to upload logs immediately after the 5th log record is added kaaClient->setLogUploadStrategy(std::make_shared<kaa::RecordCountLogUploadStrategy>(5, kaaClient->getKaaClientContext()));
#include <stdint.h> #include <kaa/kaa_logging.h> #include <kaa/platform/kaa_client.h> #include <kaa/platform-impl/common/ext_log_upload_strategies.h> kaa_client_t *kaa_client = NULL; /* * The log storage. */ void *log_storage_context = NULL; /* * Log upload strategy. */ void *log_upload_strategy_context = NULL; /* * Assume Kaa SDK and log storage are already initialized. */ kaa_log_bucket_constraints_t bucket_sizes = { /* Specify bucket size constraints */ }; /* Create a strategy based on log count. */ kaa_error_t error_code = ext_log_upload_strategy_create(kaa_client_get_context(kaa_client), &log_upload_strategy_context, THRESHOLD_COUNT_FLAG); /* Check error code */ /* After 50th log in storage, upload will be initiated. */ error_code = ext_log_upload_strategy_set_threshold_count(log_upload_strategy_context, 50); /* Check error code */ /* Initialize the log storage and strategy (by default it is not set) */ error_code = kaa_logging_init(kaa_client_get_context(kaa_client)->log_collector , log_storage_contex , log_upload_strategy_context , &bucket_sizes);
// Create log upload strategy based on number of log records RecordCountLogUploadStrategy *uploadStrategy = [[RecordCountLogUploadStrategy alloc] initWithCountThreshold:10]; // Configure client to use our newly created strategy [kaaClient setLogUploadStrategy:uploadStrategy];
Storage size strategy to upload logs after the threshold of local log storage space occupied is reached:
// Configure the strategy to upload logs every 64 KB of data collected kaaClient.setLogUploadStrategy(new StorageSizeLogUploadStrategy(64 * 1024));
// Configure the strategy to upload logs immediately after the volume of collected logs exceeds 100 bytes kaaClient->setLogUploadStrategy(std::make_shared<kaa::StorageSizeLogUploadStrategy>(100, kaaClient->getKaaClientContext()));
#include <stdint.h> #include <kaa/kaa_logging.h> #include <kaa/platform/kaa_client.h> #include <kaa/platform-impl/common/ext_log_upload_strategies.h> kaa_client_t *kaa_client = NULL; /* * The log storage. */ void *log_storage_context = NULL; /* * Log upload strategy. */ void *log_upload_strategy_context = NULL; /* * Assume Kaa SDK and log storage are already initialized. */ kaa_log_bucket_constraints_t bucket_sizes = { /* Specify bucket size constraints */ }; /* Create a strategy based on log storage volume. */ kaa_error_t error_code = ext_log_upload_strategy_create(kaa_client_get_context(kaa_client), &log_upload_strategy_context, THRESHOLD_VOLUME_FLAG); /* Check error code */ /* Set log upload strategy based on log records size (in bytes). In this case threshold will be set to 1 KB. */ error_code = ext_log_upload_strategy_set_threshold_volume(log_upload_strategy_context, 1024); /* Check error code */ /* Initialize the log storage and strategy (by default it is not set) */ error_code = kaa_logging_init(kaa_client_get_context(kaa_client)->log_collector , log_storage_contex , log_upload_strategy_context , &bucket_sizes);
// Create log upload strategy based on log records size (in bytes). In this case threshold will be set to 1 KB. StorageSizeLogUploadStrategy *uploadStrategy = [[StorageSizeLogUploadStrategy alloc] initWithVolumeThreshold:1024]; // Configure client to use our newly created strategy [kaaClient setLogUploadStrategy:uploadStrategy];
Combined periodic and log count strategy;
// Configure the strategy to upload logs every fifth log record added ... // .. OR the next log record added after 60 seconds pass since the last upload kaaClient.setLogUploadStrategy(new RecordCountWithTimeLimitLogUploadStrategy(5, 60, TimeUnit.SECONDS));
// Configure the strategy to upload logs immediately after the 5th log record is added or each 60 seconds kaaClient->setLogUploadStrategy(std::make_shared<kaa::RecordCountWithTimeLimitLogUploadStrategy>(5, 60, kaaClient->getKaaClientContext()));
#include <stdint.h> #include <kaa/kaa_logging.h> #include <kaa/platform/kaa_client.h> #include <kaa/platform-impl/common/ext_log_upload_strategies.h> kaa_client_t *kaa_client = NULL; /* * The log storage. */ void *log_storage_context = NULL; /* * Log upload strategy. */ void *log_upload_strategy_context = NULL; /* * Assume Kaa SDK and log storage are already initialized. */ kaa_log_bucket_constraints_t bucket_sizes = { /* Specify bucket size constraints */ }; /* Create a strategy based on log count and timeout. */ kaa_error_t error_code = ext_log_upload_strategy_create(kaa_client_get_context(kaa_client), &log_upload_strategy_context, KAA_LOG_UPLOAD_BY_RECORD_COUNT_AND_TIMELIMIT); /* Check error code */ /* After 50th log in storage, upload will be initiated. */ error_code = ext_log_upload_strategy_set_threshold_count(log_upload_strategy_context, 50); /* Check error code */ /* Strategy will upload logs every 5 seconds. */ error_code = ext_log_upload_strategy_set_upload_timeout(log_upload_strategy_context, 5); /* Check error code */ /* Initialize the log storage and strategy (by default it is not set) */ error_code = kaa_logging_init(kaa_client_get_context(kaa_client)->log_collector , log_storage_contex , log_upload_strategy_context , &bucket_sizes);
// Create log upload strategy, which will upload logs when either log count threshold or time limit is reached RecordCountWithTimeLimitLogUploadStrategy *uploadStrategy = [[RecordCountWithTimeLimitLogUploadStrategy alloc] initWithCountThreshold:10 timeLimit:20 timeUnit:TIME_UNIT_SECONDS]; // Configure client to use our newly created strategy [kaaClient setLogUploadStrategy:uploadStrategy];
Combined periodic and storage size strategy;
// Configure the strategy to upload logs every 8 KB of data collected ... // .. OR the next log record added after 10 seconds pass since the last upload kaaClient.setLogUploadStrategy(new StorageSizeWithTimeLimitLogUploadStrategy(8 * 1024, 10, TimeUnit.SECONDS));
// Configure the strategy to upload logs immediately after the volume of collected logs exceeds 100 bytes or each 60 seconds kaaClient->setLogUploadStrategy(std::make_shared<kaa::StorageSizeWithTimeLimitLogUploadStrategy>(100, 60, kaaClient->getKaaClientContext()));
#include <stdint.h> #include <kaa/kaa_logging.h> #include <kaa/platform/kaa_client.h> #include <kaa/platform-impl/common/ext_log_upload_strategies.h> kaa_client_t *kaa_client = NULL; /* * The log storage. */ void *log_storage_context = NULL; /* * Log upload strategy. */ void *log_upload_strategy_context = NULL; /* * Assume Kaa SDK and log storage are already initialized. */ kaa_log_bucket_constraints_t bucket_sizes = { /* Specify bucket size constraints */ }; /* Create a strategy based on log storage volume and timeout. */ kaa_error_t error_code = ext_log_upload_strategy_create(kaa_client_get_context(kaa_client), &log_upload_strategy_context, KAA_LOG_UPLOAD_BY_STORAGE_SIZE_AND_TIMELIMIT); /* Check error code */ /* Set log upload strategy based on log records size (in bytes). In this case threshold will be set to 1 KB. */ error_code = ext_log_upload_strategy_set_threshold_volume(log_upload_strategy_context, 1024); /* Check error code */ /* Strategy will upload logs every 5 seconds. */ error_code = ext_log_upload_strategy_set_upload_timeout(log_upload_strategy_context, 5); /* Check error code */ /* Initialize the log storage and strategy (by default it is not set) */ error_code = kaa_logging_init(kaa_client_get_context(kaa_client)->log_collector , log_storage_contex , log_upload_strategy_context , &bucket_sizes);
// Create log upload strategy, which will upload log records when either volume threshold (in bytes) or time limit is reached. StorageSizeWithTimeLimitLogUploadStrategy *uploadStrategy = [[StorageSizeWithTimeLimitLogUploadStrategy alloc] initWithThresholdVolume:1024 timeLimit:20 timeUnit:TIME_UNIT_SECONDS]; // Configure client to use our newly created strategy [kaaClient setLogUploadStrategy:uploadStrategy];
Combined log count and storage size strategy:
// Create an instance of the default log upload strategy DefaultLogUploadStrategy customizedStrategy = new DefaultLogUploadStrategy(); // Configure it to upload logs every fifteen log records ... strategy.setCountThreshold(15); // ... OR every 32 KB of data collected strategy.setVolumeThreshold(32 * 1024); kaaClient.setLogUploadStrategy(customizedStrategy);
// Configure the strategy to upload logs immediately after the 5th log record is added or the volume of collected logs exceeds 100 bytes auto logUploadStrategy = std::make_shared<kaa::DefaultLogUploadStrategy>(kaaClient->getKaaClientContext()); logUploadStrategy->setCountThreshold(5); logUploadStrategy->setVolumeThreshold(100); kaaClient->setLogUploadStrategy(logUploadStrategy);
#include <stdint.h> #include <kaa/kaa_logging.h> #include <kaa/platform/kaa_client.h> #include <kaa/platform-impl/common/ext_log_upload_strategies.h> kaa_client_t *kaa_client = NULL; /* * The log storage. */ void *log_storage_context = NULL; /* * Log upload strategy. */ void *log_upload_strategy_context = NULL; /* * Assume Kaa SDK and log storage are already initialized. */ kaa_log_bucket_constraints_t bucket_sizes = { /* Specify bucket size constraints */ }; /* Create a strategy based on log storage volume and count. */ kaa_error_t error_code = ext_log_upload_strategy_create(kaa_client_get_context(kaa_client), &log_upload_strategy_context, KAA_LOG_UPLOAD_VOLUME_STRATEGY); /* Check error code */ /* After 50th log in storage, upload will be initiated. */ error_code = ext_log_upload_strategy_set_threshold_count(log_upload_strategy_context, 50); /* Check error code */ /* Set log upload strategy based on log records size (in bytes). In this case threshold will be set to 1 KB. */ error_code = ext_log_upload_strategy_set_threshold_volume(log_upload_strategy_context, 1024); /* Check error code */ /* Initialize the log storage and strategy (by default it is not set) */ error_code = kaa_logging_init(kaa_client_get_context(kaa_client)->log_collector , log_storage_contex , log_upload_strategy_context , &bucket_sizes);
// Create log upload strategy, which will upload log records when either volume threshold or log count threshold is reached. DefaultLogUploadStrategy *uploadStrategy = [[DefaultLogUploadStrategy alloc] initWithDefaults]; // Set volume threshold (in bytes) for our strategy uploadStrategy.volumeThreshold = 1024; // Set log count threshold of our strategy uploadStrategy.countThreshold = 10; // Configure client to use our newly created strategy [kaaClient setLogUploadStrategy:uploadStrategy];
NOTE: This is the default behavior with log record count threshold of 64 and local storage threshold of 8 KB.
Max parallel upload strategy to limit the number of log batches sent without receiving a response from the server:
// Configure the strategy to preserve the exact order of log uploads DefaultLogUploadStrategy customizedStrategy = new DefaultLogUploadStrategy(); strategy.setMaxParallelUploads(1); kaaClient.setLogUploadStrategy(customizedStrategy);
// Configure the strategy not to upload logs until a previous upload is successfully performed auto logUploadStrategy = std::make_shared<kaa::DefaultLogUploadStrategy>(kaaClient->getKaaClientContext()); logUploadStrategy->setMaxParallelUploads(1); kaaClient->setLogUploadStrategy(logUploadStrategy);
#include <stdint.h> #include <kaa/kaa_logging.h> #include <kaa/platform/kaa_client.h> #include <kaa/platform-impl/common/ext_log_upload_strategies.h> kaa_client_t *kaa_client = NULL; /* * The log storage. */ void *log_storage_context = NULL; /* * Log upload strategy. */ void *log_upload_strategy_context = NULL; /* * Assume Kaa SDK and log storage are already initialized. */ kaa_log_bucket_constraints_t bucket_sizes = { /* Specify bucket size constraints */ }; /* Create a strategy based on log storage volume. */ kaa_error_t error_code = ext_log_upload_strategy_create(kaa_client_get_context(kaa_client), &log_upload_strategy_context, KAA_LOG_UPLOAD_VOLUME_STRATEGY); /* Check error code */ /* After 50th log in storage, upload will be initiated. */ error_code = ext_log_upload_strategy_set_threshold_count(log_upload_strategy_context, 50); /* Check error code */ /* Configure the strategy not to upload logs until a previous upload is successfully performed */ error_code = ext_log_upload_strategy_set_max_parallel_uploads(log_upload_strategy_context, 1) /* Initialize the log storage and strategy (by default it is not set) */ error_code = kaa_logging_init(kaa_client_get_context(kaa_client)->log_collector , log_storage_contex , log_upload_strategy_context , &bucket_sizes);
// Create default log upload strategy DefaultLogUploadStrategy *uploadStrategy = [[DefaultLogUploadStrategy alloc] initWithDefaults]; // Limit the maximum number of parallel log uploads uploadStrategy.maxParallelUploads = 1; // Configure client to use our newly created strategy [kaaClient setLogUploadStrategy:uploadStrategy];
An example application for collecting log data from endpoints can be found in the official Kaa repository.
Copyright © 2014-2016, CyberVision, Inc.