From 9f02e451dd2f91c217d73c8029bb939b3a211672 Mon Sep 17 00:00:00 2001 From: Naveen Bobbili Date: Sun, 28 Apr 2019 20:51:16 -0700 Subject: Code commit by ICS team to add Alexa Voice Chrome to Homescreen App. Change-Id: I2052e345baaf4306e8e3f27a01bc6940f4d27d88 Signed-off-by: Naveen Bobbili --- homescreen/homescreen.pro | 12 +- homescreen/qml/SpeechChrome.qml | 112 +++++++++++++++++ homescreen/qml/images/SpeechChrome/bar.png | Bin 0 -> 23826 bytes .../qml/images/SpeechChrome/push_to_talk.png | Bin 0 -> 1544 bytes .../qml/images/SpeechChrome/speechchrome.qrc | 6 + homescreen/qml/main.qml | 7 ++ homescreen/qml/qml.qrc | 1 + homescreen/src/aglsocketwrapper.cpp | 90 ++++++++++++++ homescreen/src/aglsocketwrapper.h | 35 ++++++ homescreen/src/chromecontroller.cpp | 137 +++++++++++++++++++++ homescreen/src/chromecontroller.h | 38 ++++++ homescreen/src/constants.h | 40 ++++++ homescreen/src/main.cpp | 4 + package/config.xml | 1 + 14 files changed, 480 insertions(+), 3 deletions(-) create mode 100644 homescreen/qml/SpeechChrome.qml create mode 100644 homescreen/qml/images/SpeechChrome/bar.png create mode 100644 homescreen/qml/images/SpeechChrome/push_to_talk.png create mode 100644 homescreen/qml/images/SpeechChrome/speechchrome.qrc create mode 100755 homescreen/src/aglsocketwrapper.cpp create mode 100755 homescreen/src/aglsocketwrapper.h create mode 100644 homescreen/src/chromecontroller.cpp create mode 100644 homescreen/src/chromecontroller.h create mode 100644 homescreen/src/constants.h diff --git a/homescreen/homescreen.pro b/homescreen/homescreen.pro index 3a25880..5240324 100644 --- a/homescreen/homescreen.pro +++ b/homescreen/homescreen.pro @@ -31,7 +31,9 @@ SOURCES += \ src/applicationlauncher.cpp \ src/mastervolume.cpp \ src/homescreenhandler.cpp \ - helpers/qafbwebsocketclient.cpp + helpers/qafbwebsocketclient.cpp \ + src/aglsocketwrapper.cpp \ + src/chromecontroller.cpp HEADERS += \ src/statusbarmodel.h \ @@ -39,7 +41,10 @@ HEADERS += \ src/applicationlauncher.h \ src/mastervolume.h \ src/homescreenhandler.h \ - helpers/qafbwebsocketclient.h + helpers/qafbwebsocketclient.h \ + src/aglsocketwrapper.h \ + src/chromecontroller.h \ + src/constants.h OTHER_FILES += \ README.md @@ -51,4 +56,5 @@ RESOURCES += \ qml/images/Shortcut/shortcut.qrc \ qml/images/Status/status.qrc \ qml/images/images.qrc \ - qml/qml.qrc + qml/qml.qrc \ + qml/images/SpeechChrome/speechchrome.qrc \ No newline at end of file diff --git a/homescreen/qml/SpeechChrome.qml b/homescreen/qml/SpeechChrome.qml new file mode 100644 index 0000000..aba8283 --- /dev/null +++ b/homescreen/qml/SpeechChrome.qml @@ -0,0 +1,112 @@ +import QtQuick 2.0 +import SpeechChrome 1.0 + +Item { + id: root + + clip: true + + Image { + id: chromeBarImage + + anchors.top: parent.top + source: "./images/SpeechChrome/bar.png" + + Behavior on x { + NumberAnimation { duration: 250 } + } + Behavior on opacity { + NumberAnimation { duration: 250 } + } + } + + Image { + id: pushToTalk + + anchors.right: parent.right + anchors.verticalCenter: parent.verticalCenter + source: "./images/SpeechChrome/push_to_talk.png" + + MouseArea { + anchors.fill: parent + onPressed: speechChromeController.pushToTalk() + } + + Behavior on opacity { + NumberAnimation { duration: 250 } + } + } + + states: [ + State { + name: "Idle" + when: speechChromeController.chromeState == SpeechChromeController.Idle + PropertyChanges { + target: chromeBarImage + opacity: 0.0 + x: 0 + } + PropertyChanges { + target: pushToTalk + opacity: 1.0 + enabled: true + } + }, + State { + name: "Listening" + when: speechChromeController.chromeState == SpeechChromeController.Listening + PropertyChanges { + target: chromeBarImage + opacity: 1.0 + x: 0 + } + PropertyChanges { + target: pushToTalk + opacity: 0.0 + enabled: false + } + }, + State { + name: "Thinking" + when: speechChromeController.chromeState == SpeechChromeController.Thinking + PropertyChanges { + target: chromeBarImage + opacity: 1.0 + x: root.width - chromeBarImage.width + } + PropertyChanges { + target: pushToTalk + opacity: 0.0 + enabled: false + } + }, + State { + name: "Speaking" + when: speechChromeController.chromeState == SpeechChromeController.Speaking + PropertyChanges { + target: chromeBarImage + opacity: 1.0 + x: (root.width - chromeBarImage.width) * 0.5 + } + PropertyChanges { + target: pushToTalk + opacity: 0.0 + enabled: false + } + }, + State { + name: "MicrophoneOff" + when: speechChromeController.chromeState == SpeechChromeController.MicrophoneOff + PropertyChanges { + target: chromeBarImage + opacity: 0.0 + x: 0 + } + PropertyChanges { + target: pushToTalk + opacity: 1.0 + enabled: true + } + } + ] +} diff --git a/homescreen/qml/images/SpeechChrome/bar.png b/homescreen/qml/images/SpeechChrome/bar.png new file mode 100644 index 0000000..caabde1 Binary files /dev/null and b/homescreen/qml/images/SpeechChrome/bar.png differ diff --git a/homescreen/qml/images/SpeechChrome/push_to_talk.png b/homescreen/qml/images/SpeechChrome/push_to_talk.png new file mode 100644 index 0000000..f45218e Binary files /dev/null and b/homescreen/qml/images/SpeechChrome/push_to_talk.png differ diff --git a/homescreen/qml/images/SpeechChrome/speechchrome.qrc b/homescreen/qml/images/SpeechChrome/speechchrome.qrc new file mode 100644 index 0000000..8ab1472 --- /dev/null +++ b/homescreen/qml/images/SpeechChrome/speechchrome.qrc @@ -0,0 +1,6 @@ + + + bar.png + push_to_talk.png + + diff --git a/homescreen/qml/main.qml b/homescreen/qml/main.qml index 7d40276..d74fd1f 100644 --- a/homescreen/qml/main.qml +++ b/homescreen/qml/main.qml @@ -90,6 +90,13 @@ Window { } } + SpeechChrome { + anchors.left: parent.left + anchors.right: parent.right + anchors.bottom: parent.bottom + height: 215 + } + Connections { target: homescreenHandler onShowInformation: { diff --git a/homescreen/qml/qml.qrc b/homescreen/qml/qml.qrc index e60ea63..d901481 100644 --- a/homescreen/qml/qml.qrc +++ b/homescreen/qml/qml.qrc @@ -10,5 +10,6 @@ StatusArea.qml TopArea.qml IconItem.qml + SpeechChrome.qml diff --git a/homescreen/src/aglsocketwrapper.cpp b/homescreen/src/aglsocketwrapper.cpp new file mode 100755 index 0000000..8352660 --- /dev/null +++ b/homescreen/src/aglsocketwrapper.cpp @@ -0,0 +1,90 @@ +#include "aglsocketwrapper.h" +#include "constants.h" + +#include +#include +#include +#include +#include + +#include + +namespace { +enum MessageTypes { + Call = 2, + Success = 3, + Error = 4, + Event = 5 +}; +} + +AglSocketWrapper::AglSocketWrapper(QObject *parent) : + QObject(parent) + , m_socket(new QWebSocket(QString(), QWebSocketProtocol::VersionLatest, this)) +{ + connect(m_socket, &QWebSocket::connected, this, &AglSocketWrapper::connected); + connect(m_socket, &QWebSocket::disconnected, this, &AglSocketWrapper::disconnected); + connect(m_socket, QOverload::of(&QWebSocket::error), + [](QAbstractSocket::SocketError error) -> void { + qWarning() << "AglSocketWrapper internal socket error" << error; + }); + connect(m_socket, &QWebSocket::textMessageReceived, + this, [this](const QString &msg) -> void { + const QJsonDocument doc = QJsonDocument::fromJson(msg.toUtf8()); + if (doc.isArray()) { + const QJsonArray msgArray = doc.array(); + if (msgArray.count() >= 3) { + const int msgType = msgArray.at(0).toInt(); + switch (msgType) { + case Success: + case Error: { + auto callbackIt = m_callbacks.find( msgArray.at(1).toString()); + if (callbackIt != m_callbacks.constEnd()) { + (*callbackIt)(msgType == Success, msgArray.at(2)); + m_callbacks.erase(callbackIt); + } + } + break; + case Event: { + const QJsonObject eventObj = msgArray.at(2).toObject(); + emit eventReceived(msgArray.at(1).toString(), eventObj.value(vshl::DATA_TAG)); + } + break; + default: + break; + } + return; + } + } + qWarning() << "Unsupported message format:" << msg; + }); +} + +void AglSocketWrapper::open(const QUrl &url) +{ + m_socket->open(url); +} + +void AglSocketWrapper::close() +{ + m_socket->close(); +} + +void AglSocketWrapper::apiCall(const QString &api, const QString &verb, const QJsonValue &args, + AglSocketWrapper::ApiCallback callback) +{ + const QString id = QUuid::createUuid().toString(); + if (callback) + m_callbacks.insert(id, callback); + + QJsonArray callData; + callData.append(Call); + callData.append(id); + callData.append(api + QLatin1String("/") + verb); + callData.append(args); + + const QString msg = QLatin1String(QJsonDocument(callData).toJson(QJsonDocument::Compact)); + m_socket->sendTextMessage(msg); + + qDebug() << Q_FUNC_INFO << "Data sent:" << msg; +} diff --git a/homescreen/src/aglsocketwrapper.h b/homescreen/src/aglsocketwrapper.h new file mode 100755 index 0000000..4807cd5 --- /dev/null +++ b/homescreen/src/aglsocketwrapper.h @@ -0,0 +1,35 @@ +#ifndef AGLSOCKETWRAPPER_H +#define AGLSOCKETWRAPPER_H + +#include +#include +#include +#include + +#include + +class QWebSocket; +class AglSocketWrapper : public QObject +{ + Q_OBJECT +public: + explicit AglSocketWrapper(QObject *parent = nullptr); + + void open(const QUrl &url); + void close(); + + using ApiCallback = std::function; + void apiCall(const QString &api, const QString &verb, const QJsonValue &args = QJsonValue(), + ApiCallback callback = nullptr); + +signals: + void connected(); + void disconnected(); + void eventReceived(const QString &eventName, const QJsonValue &data); + +private: + QWebSocket *m_socket; + QMap m_callbacks; +}; + +#endif // AGLSOCKETWRAPPER_H diff --git a/homescreen/src/chromecontroller.cpp b/homescreen/src/chromecontroller.cpp new file mode 100644 index 0000000..7994fa0 --- /dev/null +++ b/homescreen/src/chromecontroller.cpp @@ -0,0 +1,137 @@ +#include "chromecontroller.h" +#include "aglsocketwrapper.h" +#include "constants.h" + +#include +#include +#include + +ChromeController::ChromeController(const QUrl &bindingUrl, QObject *parent) : + QObject(parent) + , m_aglSocket(new AglSocketWrapper(this)) +{ + //Alexa voice agent subscription---------------------------------------------------------------- + { + connect(m_aglSocket, &AglSocketWrapper::connected, + this, [this]() -> void { + m_aglSocket->apiCall(vshl::API, vshl::VOICE_AGENT_ENUMERATION_VERB, QJsonValue(), + [this](bool result, const QJsonValue &data) -> void { + if (!result) { + qWarning() << "Failed to enumerate voice agents"; + return; + } + + QJsonObject dataObj = data.toObject(); + auto objIt = dataObj.find(vshl::RESPONSE_TAG); + if (objIt == dataObj.constEnd()) { + qWarning() << "Voice agent enumeration response tag missing." + << dataObj; + return; + } + + dataObj = objIt.value().toObject(); + objIt = dataObj.find(vshl::AGENTS_TAG); + if (objIt == dataObj.constEnd()) { + qWarning() << "Voice agent enumeration agents tag missing." + << dataObj; + return; + } + + const QJsonArray agents = objIt.value().toArray(); + QString alexaAgentId; + for (const QJsonValue &agent : agents) { + const QJsonObject agentObj = agent.toObject(); + auto agentIt = agentObj.find(vshl::NAME_TAG); + if (agentIt == agentObj.constEnd()) + continue; + const QString agentName = agentIt.value().toString(); + agentIt = agentObj.find(vshl::ID_TAG); + if (agentIt == agentObj.constEnd()) + continue; + if (agentName.compare(vshl::ALEXA_AGENT_NAME) == 0) { + alexaAgentId = agentIt.value().toString(); + break; + } + } + if (alexaAgentId.isEmpty()) { + qWarning() << "Alexa voice agent not found"; + return; + } + + //Voice agent subscription------------------------------------------------------ + { + m_voiceAgentId = alexaAgentId; + const QJsonObject args { + { vshl::VOICE_AGENT_ID_ARG, alexaAgentId }, + { vshl::VOICE_AGENT_EVENTS_ARG, vshl::ALEXA_EVENTS_ARRAY } + }; + m_aglSocket->apiCall(vshl::API, vshl::SUBSCRIBE_VERB, args, + [](bool result, const QJsonValue &data) -> void { + qDebug() << (vshl::API + QLatin1String(":") + vshl::SUBSCRIBE_VERB) + << "result: " << result << " val: " << data; + }); + } + //------------------------------------------------------------------------------ + }); + }); + } + //----------------------------------------------------------------------------------------------< + + //Socket connection management------------------------------------------------------------------ + { + auto connectToBinding = [bindingUrl, this]() -> void { + m_aglSocket->open(bindingUrl); + qDebug() << "Connecting to:" << bindingUrl; + }; + connect(m_aglSocket, &AglSocketWrapper::disconnected, this, [connectToBinding]() -> void { + QTimer::singleShot(2500, connectToBinding); + }); + connectToBinding(); + } + //---------------------------------------------------------------------------------------------- + + //Speech chrome state change event handling----------------------------------------------------- + { + connect(m_aglSocket, &AglSocketWrapper::eventReceived, + this, [this](const QString &eventName, const QJsonValue &data) -> void { + if (eventName.compare(vshl::VOICE_DIALOG_STATE_EVENT + m_voiceAgentId) == 0) { + const QJsonObject dataObj = QJsonDocument::fromJson(data.toString().toUtf8()).object(); + auto objIt = dataObj.find(vshl::STATE_TAG); + if (objIt == dataObj.constEnd()) { + qWarning() << "Voice dialog state event state missing."; + return; + } + const QString stateStr = objIt.value().toString(); + if (stateStr.compare(vshl::VOICE_DIALOG_IDLE) == 0) { + setChromeState(Idle); + } else if (stateStr.compare(vshl::VOICE_DIALOG_LISTENING) == 0) { + setChromeState(Listening); + } else if (stateStr.compare(vshl::VOICE_DIALOG_THINKING) == 0) { + setChromeState(Thinking); + } else if (stateStr.compare(vshl::VOICE_DIALOG_SPEAKING) == 0) { + setChromeState(Speaking); + } else if (stateStr.compare(vshl::VOICE_DIALOG_MICROPHONEOFF) == 0) { + setChromeState(MicrophoneOff); + } + } + }); + } + //---------------------------------------------------------------------------------------------- +} + +void ChromeController::pushToTalk() +{ + m_aglSocket->apiCall(vshl::API, vshl::TAP_TO_TALK_VERB, QJsonValue(), + [](bool result, const QJsonValue &data) -> void { + qDebug() << (vshl::API + QLatin1String(":") + vshl::TAP_TO_TALK_VERB) + << "result: " << result << " val: " << data; + }); +} + +void ChromeController::setChromeState(ChromeController::ChromeState state) +{ + if (m_chromeState != state) { + m_chromeState = state; + emit chromeStateChanged(); + } +} diff --git a/homescreen/src/chromecontroller.h b/homescreen/src/chromecontroller.h new file mode 100644 index 0000000..27c26cb --- /dev/null +++ b/homescreen/src/chromecontroller.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include + +class AglSocketWrapper; +class ChromeController : public QObject +{ + Q_OBJECT + + Q_PROPERTY(int chromeState READ chromeState NOTIFY chromeStateChanged) + +public: + enum ChromeState { + Idle = 0, + Listening, + Thinking, + Speaking, + MicrophoneOff + }; + Q_ENUM(ChromeState) + + explicit ChromeController(const QUrl &bindingUrl, QObject *parent = nullptr); + int chromeState() const { return m_chromeState; } + +public slots: + void pushToTalk(); + +signals: + void chromeStateChanged(); + +private: + void setChromeState(ChromeState state); + + AglSocketWrapper *m_aglSocket; + QString m_voiceAgentId; + ChromeState m_chromeState = Idle; +}; diff --git a/homescreen/src/constants.h b/homescreen/src/constants.h new file mode 100644 index 0000000..38de9b9 --- /dev/null +++ b/homescreen/src/constants.h @@ -0,0 +1,40 @@ +#ifndef CONSTANTS_H +#define CONSTANTS_H + +#include +#include +#include + +namespace vshl { +const QString API = QLatin1String("vshl-core"); +const QString VOICE_AGENT_ENUMERATION_VERB = QLatin1String("enumerateVoiceAgents"); +const QString SUBSCRIBE_VERB = QLatin1String("subscribe"); +const QString TAP_TO_TALK_VERB = QLatin1String("startListening"); + +const QString ALEXA_AGENT_NAME = QLatin1String("Alexa"); +const QJsonArray ALEXA_EVENTS_ARRAY = { + QLatin1String("voice_authstate_event"), + QLatin1String("voice_dialogstate_event"), + QLatin1String("voice_connectionstate_event") +}; + +const QString DATA_TAG = QLatin1String("data"); +const QString RESPONSE_TAG = QLatin1String("response"); +const QString AGENTS_TAG = QLatin1String("agents"); +const QString NAME_TAG = QLatin1String("name"); +const QString ID_TAG = QLatin1String("id"); +const QString STATE_TAG = QLatin1String("state"); + +const QString VOICE_AGENT_ID_ARG = QLatin1String("va_id"); +const QString VOICE_AGENT_EVENTS_ARG = QLatin1String("events"); +const QString VOICE_AGENT_ACTIONS_ARG = QLatin1String("actions"); + +const QString VOICE_DIALOG_STATE_EVENT = QLatin1String("vshl-core/voice_dialogstate_event#"); +const QString VOICE_DIALOG_IDLE = QLatin1String("IDLE"); +const QString VOICE_DIALOG_LISTENING = QLatin1String("LISTENING"); +const QString VOICE_DIALOG_THINKING = QLatin1String("THINKING"); +const QString VOICE_DIALOG_SPEAKING = QLatin1String("SPEAKING"); +const QString VOICE_DIALOG_MICROPHONEOFF = QLatin1String("MICROPHONEOFF"); +} + +#endif // CONSTANTS_H diff --git a/homescreen/src/main.cpp b/homescreen/src/main.cpp index 5f283fb..5c819f9 100644 --- a/homescreen/src/main.cpp +++ b/homescreen/src/main.cpp @@ -32,6 +32,7 @@ #include "mastervolume.h" #include "homescreenhandler.h" #include "hmi-debug.h" +#include "chromecontroller.h" // XXX: We want this DBus connection to be shared across the different // QML objects, is there another way to do this, a nice way, perhaps? @@ -91,6 +92,8 @@ int main(int argc, char *argv[]) // qmlRegisterType("HomeScreen", 1, 0, "ApplicationLauncher"); qmlRegisterType("HomeScreen", 1, 0, "StatusBarModel"); qmlRegisterType("MasterVolume", 1, 0, "MasterVolume"); + qmlRegisterUncreatableType("SpeechChrome", 1, 0, "SpeechChromeController", + QLatin1String("SpeechChromeController is uncreatable.")); ApplicationLauncher *launcher = new ApplicationLauncher(); QLibWindowmanager* layoutHandler = new QLibWindowmanager(); @@ -140,6 +143,7 @@ int main(int argc, char *argv[]) engine.rootContext()->setContextProperty("launcher", launcher); engine.rootContext()->setContextProperty("weather", new Weather(bindingAddress)); engine.rootContext()->setContextProperty("bluetooth", new Bluetooth(bindingAddress, engine.rootContext())); + engine.rootContext()->setContextProperty("speechChromeController", new ChromeController(bindingAddress, &engine)); engine.rootContext()->setContextProperty("screenInfo", &screenInfo); engine.load(QUrl(QStringLiteral("qrc:/main.qml"))); diff --git a/package/config.xml b/package/config.xml index a15baee..56d9b40 100644 --- a/package/config.xml +++ b/package/config.xml @@ -13,6 +13,7 @@ + -- cgit