Rework to remove Alexa voiceagent dependencieshalibut_8.0.4halibut_8.0.3halibut/8.0.4halibut/8.0.38.0.48.0.3

Changes include:
- The previous somewhat hard-coded Alexa voiceagent configuration in the
  app controller definition has been replaced with a scheme that allows
  instead specifying voiceagent configuration via the configuration file
  /etc/xdg/AGL/voice-high.json, and individual per-voiceagent
  configuration via JSON files in the directory
  /etc/xdg/AGL/voiceagents. See the updated README.md for more details.
- The new file reading code for the above change was facilitated by
  bumping the C++ standard option to 17 (from 11) to allow use of the
  new standard library filesystem classes.
- The configured voiceagents' APIs are now loaded dynamically using the
  afb_api_require_api call enabled by the recently exposed
  platform:apis:auto-ws widget permission. The widget configuration has
  been updated to add the permission, and also remove the previously
  hard-code dependency on the "alexa-voiceagent" API.
- App controller event hooks are created dynamically for the events from
  the configured voiceagents. This is done by calling back into the
  app controller API at the appropriate place during initialization,
  replacing the previous hard-coding of events from "alexa-voiceagent".
  If future use cases require dynamic removal of voiceagents, then this
  scheme will need to be revisited, as the app controller API currently
  has no provision for removing events.
- The IAFBApi interdace and AFBApiImpl implementation classes have been
  extended with additional getApi and requireApi calls to enable the
  above changes in a straightforward manner.

Bug-AGL: SPEC-2898

Signed-off-by: Scott Murray <scott.murray@konsulko.com>
Change-Id: Iae8b99a6286174510e88d8eeffd51718db64f2f4
(cherry picked from commit d75ca330180e943df28435705d36c9e435759de2)

7 files changed, 214 insertions, 61 deletions

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index b5f8d35..8bb94fc 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -17,7 +17,7 @@
 
 # Add target to project dependency list
 PROJECT_TARGET_ADD(vshl-core)
-    set(CMAKE_CXX_STANDARD 11)
+    set(CMAKE_CXX_STANDARD 17)
     set(CMAKE_CXX_STANDARD_REQUIRED ON)
     set(CMAKE_CXX_EXTENSIONS OFF)
 
diff --git a/src/plugins/CMakeLists.txt b/src/plugins/CMakeLists.txt
index 64f2c35..d6dc250 100644
--- a/src/plugins/CMakeLists.txt
+++ b/src/plugins/CMakeLists.txt
@@ -17,7 +17,7 @@
 
 PROJECT_TARGET_ADD(vshl-core-api)
 
-    set(CMAKE_CXX_STANDARD 11)
+    set(CMAKE_CXX_STANDARD 17)
     set(CMAKE_CXX_STANDARD_REQUIRED ON)
     set(CMAKE_CXX_EXTENSIONS OFF)
 
@@ -91,6 +91,7 @@ PROJECT_TARGET_ADD(vshl-core-api)
     TARGET_LINK_LIBRARIES(${TARGET_NAME}
         ${GLIB_PKG_LIBRARIES}
         ${link_libraries}
+        -lstdc++fs
     )
 
     option(ENABLE_UNIT_TESTS "Build unit tests or not" OFF)
diff --git a/src/plugins/VshlCoreApi.cpp b/src/plugins/VshlCoreApi.cpp
index 05c894d..4c32391 100644
--- a/src/plugins/VshlCoreApi.cpp
+++ b/src/plugins/VshlCoreApi.cpp
@@ -17,7 +17,10 @@
 
 #include <list>
 #include <sstream>
+#include <fstream>
+#include <filesystem>
 #include <json.hpp>
+#include <json-c/json.h>
 
 #include "afb/AFBApiImpl.h"
 #include "afb/AFBRequestImpl.h"
@@ -49,6 +52,10 @@ static std::string STARTLISTENING_JSON_ATTR_REQUEST = "request_id";
 static std::string EVENTS_JSON_ATTR_VA_ID = "va_id";
 static std::string EVENTS_JSON_ATTR_EVENTS = "events";
 
+static std::string CONFIG_JSON_FILE = "/etc/xdg/AGL/voice-high.json";
+static std::string JSON_EXT = ".json";
+static std::string VA_CONFIG_JSON_DIR = "/etc/xdg/AGL/voiceagents";
+
 static std::shared_ptr<vshlcore::utilities::logging::Logger> sLogger;
 static std::shared_ptr<vshlcore::common::interfaces::IAFBApi> sAfbApi;
 static std::unique_ptr<vshlcore::core::VRRequestProcessor> sVRRequestProcessor;
@@ -151,69 +158,198 @@ CTLP_CAPI(onDialogStateEvent, source, argsJ, eventJ) {
     return 0;
 }
 
-CTLP_CAPI(loadVoiceAgentsConfig, source, argsJ, eventJ) {
-    if (sVoiceAgentsDataManager == nullptr) {
-        sLogger->log(Level::WARNING, TAG, "loadVoiceAgentsConfig: Voice service not initialized.");
+// Helper function to add events for voice agent to controller
+// configuration.  It relies on the controller configuration being
+// available via the userdata of the API pointer.
+static int AddVoiceAgentEvents(std::string &agentApi) {
+    // Retrieve section config from api handle
+    CtlConfigT *ctrlConfig = (CtlConfigT*) afb_api_get_userdata(sAfbApi->getApi());
+    if (ctrlConfig == nullptr) {
+        sLogger->log(Level::ERROR, TAG, "AddVoiceAgentEvents: ctrlConfig == nullptr");
         return -1;
     }
 
-    if (argsJ == nullptr) {
-        sLogger->log(Level::WARNING, TAG, "loadVoiceAgentsConfig: No arguments supplied.");
+    CtlSectionT* section = nullptr;
+    for (int idx = 0; ctrlConfig->sections[idx].key != NULL; ++idx) {
+        if (!strcasecmp(ctrlConfig->sections[idx].key, "events")) {
+            section = &(ctrlConfig->sections[idx]);
+            break;
+        }
+    }
+    if (section == nullptr) {
+        sLogger->log(Level::ERROR, TAG, "AddVoiceAgentEvents: events section not found");
         return -1;
     }
 
-    json agentsConfigJson = json::parse(json_object_to_json_string(argsJ));
-    if (agentsConfigJson.find(VA_JSON_ATTR_AGENTS) == agentsConfigJson.end()) {
-        sLogger->log(Level::ERROR, TAG, "loadVoiceAgentsConfig: No agents object found in agents json");
-        return -1;
+    // Fill out events JSON array to pass to the controller
+    json_object* eventsJ = json_object_new_array();
+
+    json_object *eventJ = json_object_new_object();
+    std::string uid = agentApi + "/voice_authstate_event";
+    json_object *uidJ = json_object_new_string(uid.c_str());
+    json_object_object_add(eventJ, "uid", uidJ);
+    json_object *actionJ = json_object_new_string("plugin://vshl-core#onAuthStateEvent");
+    json_object_object_add(eventJ, "action", actionJ);
+    json_object_array_add(eventsJ, eventJ);
+
+    eventJ = json_object_new_object();
+    uid = agentApi + "/voice_connectionstate_event";
+    uidJ = json_object_new_string(uid.c_str());
+    json_object_object_add(eventJ, "uid", uidJ);
+    actionJ = json_object_new_string("plugin://vshl-core#onConnectionStateEvent");
+    json_object_object_add(eventJ, "action", actionJ);
+    json_object_array_add(eventsJ, eventJ);
+
+    eventJ = json_object_new_object();
+    uid = agentApi + "/voice_dialogstate_event";
+    uidJ = json_object_new_string(uid.c_str());
+    json_object_object_add(eventJ, "uid", uidJ);
+    actionJ = json_object_new_string("plugin://vshl-core#onDialogStateEvent");
+    json_object_object_add(eventJ, "action", actionJ);
+    json_object_array_add(eventsJ, eventJ);
+
+    // Call into controller to add event actions
+    // NOTE: AFAICT the JSON objects end up reused by the controller data
+    //       structures, so eventsJ should not be freed with a put after
+    //       this call.
+    int rc = AddActionsToSection(sAfbApi->getApi(), section, eventsJ, 0);
+    if (rc != 0) {
+        stringstream message;
+        message << "AddVoiceAgentEvents: AddActionsToSection rc = " << rc;
+        sLogger->log(Level::WARNING, TAG, message.str().c_str());
     }
+    return rc;
+}
 
-    json agentsJson = agentsConfigJson[VA_JSON_ATTR_AGENTS];
-    for (auto agentIt = agentsJson.begin(); agentIt != agentsJson.end(); ++agentIt) {
-        json agentJson = *agentIt;
-
-        if (agentJson.find(VA_JSON_ATTR_ID) == agentJson.end() ||
-            agentJson.find(VA_JSON_ATTR_ACTIVE) == agentJson.end() ||
-            agentJson.find(VA_JSON_ATTR_NAME) == agentJson.end() ||
-            agentJson.find(VA_JSON_ATTR_API) == agentJson.end() ||
-            agentJson.find(VA_JSON_ATTR_WWS) == agentJson.end() ||
-            agentJson.find(VA_JSON_ATTR_ACTIVE_WW) == agentJson.end() ||
-            agentJson.find(VA_JSON_ATTR_DESCRIPTION) == agentJson.end() ||
-            agentJson.find(VA_JSON_ATTR_VENDOR) == agentJson.end()) {
-            std::stringstream error;
-            error << "loadVoiceAgentsConfig: One or more missing params in agent "
-                     "config "
-                  << agentJson.dump();
-            sLogger->log(Level::WARNING, TAG, error.str().c_str());
-            continue;
-        }
+// Helper function to parse a voiceagent's configuration
+static bool parseVoiceAgentConfig(const json &agentJson) {
+    if (agentJson.find(VA_JSON_ATTR_ID) == agentJson.end() ||
+	agentJson.find(VA_JSON_ATTR_ACTIVE) == agentJson.end() ||
+	agentJson.find(VA_JSON_ATTR_NAME) == agentJson.end() ||
+	agentJson.find(VA_JSON_ATTR_API) == agentJson.end() ||
+	agentJson.find(VA_JSON_ATTR_WWS) == agentJson.end() ||
+	agentJson.find(VA_JSON_ATTR_ACTIVE_WW) == agentJson.end() ||
+	agentJson.find(VA_JSON_ATTR_DESCRIPTION) == agentJson.end() ||
+	agentJson.find(VA_JSON_ATTR_VENDOR) == agentJson.end()) {
+	stringstream error;
+	error << "loadVoiceAgentConfig: One or more missing params in agent "
+	    "config "
+	      << agentJson.dump();
+	sLogger->log(Level::WARNING, TAG, error.str().c_str());
+	return false;
+    }
 
-        std::string id(agentJson[VA_JSON_ATTR_ID].get<string>());
-        std::string name(agentJson[VA_JSON_ATTR_NAME].get<string>());
-        std::string api(agentJson[VA_JSON_ATTR_API].get<string>());
-        std::string description(agentJson[VA_JSON_ATTR_DESCRIPTION].get<string>());
-        std::string vendor(agentJson[VA_JSON_ATTR_VENDOR].get<string>());
-        std::string activeWakeword(agentJson[VA_JSON_ATTR_ACTIVE_WW].get<string>());
-        bool isActive(agentJson[VA_JSON_ATTR_ACTIVE].get<bool>());
-
-        shared_ptr<unordered_set<string>> wakewords = std::make_shared<unordered_set<string>>();
-        json wakewordsJson = agentJson[VA_JSON_ATTR_WWS];
-        for (auto wwIt = wakewordsJson.begin(); wwIt != wakewordsJson.end(); ++wwIt) {
-            wakewords->insert(wwIt->get<string>());
-        }
+    std::string id(agentJson[VA_JSON_ATTR_ID].get<string>());
+    std::string name(agentJson[VA_JSON_ATTR_NAME].get<string>());
+    std::string api(agentJson[VA_JSON_ATTR_API].get<string>());
+    std::string description(agentJson[VA_JSON_ATTR_DESCRIPTION].get<string>());
+    std::string vendor(agentJson[VA_JSON_ATTR_VENDOR].get<string>());
+    std::string activeWakeword(agentJson[VA_JSON_ATTR_ACTIVE_WW].get<string>());
+    bool isActive(agentJson[VA_JSON_ATTR_ACTIVE].get<bool>());
+
+    shared_ptr<unordered_set<string>> wakewords = std::make_shared<unordered_set<string>>();
+    json wakewordsJson = agentJson[VA_JSON_ATTR_WWS];
+    for (auto wwIt = wakewordsJson.begin(); wwIt != wakewordsJson.end(); ++wwIt) {
+	wakewords->insert(wwIt->get<string>());
+    }
+
+    if (sAfbApi->requireApi(api, true) != 0) {
+	stringstream error;
+	error << "loadVoiceAgentConfig: Failed to require API \""
+	      << api
+	      << "\" for voiceagent \""
+	      << name
+	      << "\".";
+	sLogger->log(Level::ERROR, TAG, error.str().c_str());
+	return false;
+    }
+
+    if (AddVoiceAgentEvents(api) != 0) {
+	stringstream error;
+	error << "loadVoiceAgentConfig: Failed to add events for voiceagent \""
+	      << name
+	      << "\".";
+	sLogger->log(Level::ERROR, TAG, error.str().c_str());
+	return false;
+    }
+
+    return sVoiceAgentsDataManager->addNewVoiceAgent(
+	id, name, description, api, vendor, activeWakeword, isActive, wakewords);
+}
 
-        sVoiceAgentsDataManager->addNewVoiceAgent(
-            id, name, description, api, vendor, activeWakeword, isActive, wakewords);
+// Helper function to parse voiceagents configuration
+static void parseVoiceAgentsConfig(const json &agentsConfigJson, std::string &defaultVoiceAgent) {
+    if (agentsConfigJson.find(VA_JSON_ATTR_AGENTS) != agentsConfigJson.end()) {
+	json agentsJson = agentsConfigJson[VA_JSON_ATTR_AGENTS];
+	for (auto agentIt = agentsJson.begin(); agentIt != agentsJson.end(); ++agentIt) {
+	    json agentJson = *agentIt;
+	    parseVoiceAgentConfig(agentJson);
+	}
     }
 
-    // Set the default agent.
-    if (agentsConfigJson.find(VA_JSON_ATTR_DEFAULT) == agentsConfigJson.end()) {
-        sLogger->log(Level::ERROR, TAG, "loadVoiceAgentsConfig: No default agent found in agents json");
+    // Save the default agent if specified
+    if (agentsConfigJson.find(VA_JSON_ATTR_DEFAULT) != agentsConfigJson.end()) {
+	defaultVoiceAgent = agentsConfigJson[VA_JSON_ATTR_DEFAULT].get<string>();
+    }
+}
+
+CTLP_CAPI(loadVoiceAgentsConfig, source, argsJ, eventJ) {
+    if (sVoiceAgentsDataManager == nullptr) {
+        sLogger->log(Level::WARNING, TAG, "loadVoiceAgentsConfig: Voice service not initialized.");
         return -1;
     }
-    std::string defaultAgentId(agentsConfigJson[VA_JSON_ATTR_DEFAULT].get<string>());
-    sVoiceAgentsDataManager->setDefaultVoiceAgent(defaultAgentId);
 
+    string defaultVoiceAgent;
+
+    if (argsJ != nullptr) {
+	// Parse any agent configs from controller arguments
+	json agentsConfigJson = json::parse(json_object_to_json_string(argsJ));
+	parseVoiceAgentsConfig(agentsConfigJson, defaultVoiceAgent);
+    }
+
+    // Load external configuration if present
+    filesystem::path agentsConfigFile = CONFIG_JSON_FILE;
+    if(filesystem::exists(agentsConfigFile)) {
+	stringstream message;
+	message << "loadVoiceAgentsConfig: reading " << agentsConfigFile.string();;
+	sLogger->log(Level::INFO, TAG, message.str().c_str());
+
+	ifstream agentsConfigJsonFile(agentsConfigFile.string());
+	json agentsConfigJson;
+	agentsConfigJsonFile >> agentsConfigJson;
+	parseVoiceAgentsConfig(agentsConfigJson, defaultVoiceAgent);
+    }
+
+    // Load any external individual voiceagent configurations
+    error_code ec;
+    for (const auto& entry : filesystem::directory_iterator(VA_CONFIG_JSON_DIR, ec)) {
+	if (entry.is_regular_file(ec) &&
+	    entry.path().extension().string() == JSON_EXT) {
+	    const auto filename = entry.path().string();
+
+	    stringstream message;
+	    message << "loadVoiceAgentsConfig: reading " << filename;
+	    sLogger->log(Level::INFO, TAG, message.str().c_str());
+
+	    ifstream jsonFile(filename);
+	    json agentJson;
+	    jsonFile >> agentJson;
+	    parseVoiceAgentConfig(agentJson);
+	}
+    }
+
+    // Set the default agent if specified
+    if (!defaultVoiceAgent.empty()) {
+	stringstream message;
+	message << "loadVoiceAgentsConfig: setting default agent to " << defaultVoiceAgent;
+	sLogger->log(Level::INFO, TAG, message.str().c_str());
+
+	sVoiceAgentsDataManager->setDefaultVoiceAgent(defaultVoiceAgent);
+    } else {
+        sLogger->log(Level::WARNING, TAG, "loadVoiceAgentsConfig: No default agent found");
+    }
+
+    // Always return zero so service will start
     return 0;
 }
 
diff --git a/src/plugins/VshlCoreApi.h b/src/plugins/VshlCoreApi.h
index 557cecc..5e56b75 100644
--- a/src/plugins/VshlCoreApi.h
+++ b/src/plugins/VshlCoreApi.h
@@ -1,5 +1,6 @@
 /*
  * Copyright 2017-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * Copyright 2019 Konsulo Group
  *
  * Licensed under the Apache License, Version 2.0 (the "License").
  * You may not use this file except in compliance with the License.
@@ -15,7 +16,7 @@
 #ifndef VSHL_CORE_API_INCLUDE
 #define VSHL_CORE_API_INCLUDE
 
-#include "ctl-plugin.h"
+#include "ctl-config.h"
 
 #ifdef __cplusplus
 extern "C" {
diff --git a/src/plugins/afb/AFBApiImpl.cpp b/src/plugins/afb/AFBApiImpl.cpp
index c4cc1b1..d7c1fcf 100644
--- a/src/plugins/afb/AFBApiImpl.cpp
+++ b/src/plugins/afb/AFBApiImpl.cpp
@@ -19,12 +19,6 @@
 #include "afb/include/AFBEventImpl.h"
 #include "utilities/logging/Logger.h"
 
-extern "C" {
-#define AFB_BINDING_VERSION 3
-
-#include <afb/afb-binding.h>
-}
-
 static std::string TAG = "vshlcore::afb::AFBApiImpl";
 
 /**
@@ -74,5 +68,11 @@ int AFBApiImpl::callSync(
     return rc;
 }
 
+int AFBApiImpl::requireApi(
+    const std::string& api,
+    const bool initialize) {
+    return afb_api_require_api(mApi, api.c_str(), initialize);
+}
+
 }  // namespace afb
 }  // namespace vshl
diff --git a/src/plugins/afb/AFBApiImpl.h b/src/plugins/afb/AFBApiImpl.h
index c0c7dd5..2ffa4a5 100644
--- a/src/plugins/afb/AFBApiImpl.h
+++ b/src/plugins/afb/AFBApiImpl.h
@@ -18,10 +18,6 @@
 
 #include <memory>
 
-extern "C" {
-#include "ctl-plugin.h"
-}
-
 #include "interfaces/afb/IAFBApi.h"
 #include "interfaces/utilities/logging/ILogger.h"
 
@@ -35,6 +31,8 @@ public:
 
     ~AFBApiImpl();
 
+    afb_api_t getApi() { return mApi; };
+
     std::shared_ptr<IAFBEvent> createEvent(const std::string& eventName) override;
 
     int callSync(
@@ -45,6 +43,10 @@ public:
         std::string& error,
         std::string& info) override;
 
+    int requireApi(
+        const std::string& api,
+        const bool initialize) override;
+
 private:
     AFBApiImpl(afb_api_t api);
 
diff --git a/src/plugins/interfaces/afb/IAFBApi.h b/src/plugins/interfaces/afb/IAFBApi.h
index ba0de72..32e459e 100644
--- a/src/plugins/interfaces/afb/IAFBApi.h
+++ b/src/plugins/interfaces/afb/IAFBApi.h
@@ -1,5 +1,6 @@
 /*
  * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * Copyright 2019 Konsulo Group
  *
  * Licensed under the Apache License, Version 2.0 (the "License").
  * You may not use this file except in compliance with the License.
@@ -18,6 +19,12 @@
 #include <memory>
 #include <string>
 
+extern "C" {
+#define AFB_BINDING_VERSION 3
+
+#include <afb/afb-binding.h>
+}
+
 #include <json-c/json_object.h>
 
 using namespace std;
@@ -80,6 +87,8 @@ public:
         virtual bool unsubscribe(IAFBRequest& request) = 0;
     };
 
+    virtual afb_api_t getApi() = 0;
+
     virtual std::shared_ptr<IAFBEvent> createEvent(const std::string& eventName) = 0;
 
     virtual int callSync(
@@ -89,6 +98,10 @@ public:
         struct json_object** result,
         std::string& error,
         std::string& info) = 0;
+
+    virtual int requireApi(
+		   const std::string& api,
+		   const bool initialize) = 0;
 };
 
 }  // namespace interfaces