aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorNaveen Bobbili <nbobbili@amazon.com>2019-04-28 20:51:16 -0700
committerJan-Simon Möller <jsmoeller@linuxfoundation.org>2019-06-24 14:14:42 +0200
commit9f02e451dd2f91c217d73c8029bb939b3a211672 (patch)
treed79eb0f93e835746169b3a96454cf8da70c0c037
parent2509e623a5f95a612a715818f15daddf2ca77f12 (diff)
Code commit by ICS team to add Alexa Voice Chrome to Homescreen App.sandbox/jsmoeller/speechdemo
Change-Id: I2052e345baaf4306e8e3f27a01bc6940f4d27d88 Signed-off-by: Naveen Bobbili <nbobbili@amazon.com>
-rw-r--r--homescreen/homescreen.pro12
-rw-r--r--homescreen/qml/SpeechChrome.qml112
-rw-r--r--homescreen/qml/images/SpeechChrome/bar.pngbin0 -> 23826 bytes
-rw-r--r--homescreen/qml/images/SpeechChrome/push_to_talk.pngbin0 -> 1544 bytes
-rw-r--r--homescreen/qml/images/SpeechChrome/speechchrome.qrc6
-rw-r--r--homescreen/qml/main.qml7
-rw-r--r--homescreen/qml/qml.qrc1
-rwxr-xr-xhomescreen/src/aglsocketwrapper.cpp90
-rwxr-xr-xhomescreen/src/aglsocketwrapper.h35
-rw-r--r--homescreen/src/chromecontroller.cpp137
-rw-r--r--homescreen/src/chromecontroller.h38
-rw-r--r--homescreen/src/constants.h40
-rw-r--r--homescreen/src/main.cpp4
-rw-r--r--package/config.xml1
14 files changed, 480 insertions, 3 deletions
diff --git a/homescreen/homescreen.pro b/homescreen/homescreen.pro
index 3a25880..5240324 100644
--- a/homescreen/homescreen.pro
+++ b/homescreen/homescreen.pro
@@ -31,7 +31,9 @@ SOURCES += \
src/applicationlauncher.cpp \
src/mastervolume.cpp \
src/homescreenhandler.cpp \
- helpers/qafbwebsocketclient.cpp
+ helpers/qafbwebsocketclient.cpp \
+ src/aglsocketwrapper.cpp \
+ src/chromecontroller.cpp
HEADERS += \
src/statusbarmodel.h \
@@ -39,7 +41,10 @@ HEADERS += \
src/applicationlauncher.h \
src/mastervolume.h \
src/homescreenhandler.h \
- helpers/qafbwebsocketclient.h
+ helpers/qafbwebsocketclient.h \
+ src/aglsocketwrapper.h \
+ src/chromecontroller.h \
+ src/constants.h
OTHER_FILES += \
README.md
@@ -51,4 +56,5 @@ RESOURCES += \
qml/images/Shortcut/shortcut.qrc \
qml/images/Status/status.qrc \
qml/images/images.qrc \
- qml/qml.qrc
+ qml/qml.qrc \
+ qml/images/SpeechChrome/speechchrome.qrc \ No newline at end of file
diff --git a/homescreen/qml/SpeechChrome.qml b/homescreen/qml/SpeechChrome.qml
new file mode 100644
index 0000000..aba8283
--- /dev/null
+++ b/homescreen/qml/SpeechChrome.qml
@@ -0,0 +1,112 @@
+import QtQuick 2.0
+import SpeechChrome 1.0
+
+Item {
+ id: root
+
+ clip: true
+
+ Image {
+ id: chromeBarImage
+
+ anchors.top: parent.top
+ source: "./images/SpeechChrome/bar.png"
+
+ Behavior on x {
+ NumberAnimation { duration: 250 }
+ }
+ Behavior on opacity {
+ NumberAnimation { duration: 250 }
+ }
+ }
+
+ Image {
+ id: pushToTalk
+
+ anchors.right: parent.right
+ anchors.verticalCenter: parent.verticalCenter
+ source: "./images/SpeechChrome/push_to_talk.png"
+
+ MouseArea {
+ anchors.fill: parent
+ onPressed: speechChromeController.pushToTalk()
+ }
+
+ Behavior on opacity {
+ NumberAnimation { duration: 250 }
+ }
+ }
+
+ states: [
+ State {
+ name: "Idle"
+ when: speechChromeController.chromeState == SpeechChromeController.Idle
+ PropertyChanges {
+ target: chromeBarImage
+ opacity: 0.0
+ x: 0
+ }
+ PropertyChanges {
+ target: pushToTalk
+ opacity: 1.0
+ enabled: true
+ }
+ },
+ State {
+ name: "Listening"
+ when: speechChromeController.chromeState == SpeechChromeController.Listening
+ PropertyChanges {
+ target: chromeBarImage
+ opacity: 1.0
+ x: 0
+ }
+ PropertyChanges {
+ target: pushToTalk
+ opacity: 0.0
+ enabled: false
+ }
+ },
+ State {
+ name: "Thinking"
+ when: speechChromeController.chromeState == SpeechChromeController.Thinking
+ PropertyChanges {
+ target: chromeBarImage
+ opacity: 1.0
+ x: root.width - chromeBarImage.width
+ }
+ PropertyChanges {
+ target: pushToTalk
+ opacity: 0.0
+ enabled: false
+ }
+ },
+ State {
+ name: "Speaking"
+ when: speechChromeController.chromeState == SpeechChromeController.Speaking
+ PropertyChanges {
+ target: chromeBarImage
+ opacity: 1.0
+ x: (root.width - chromeBarImage.width) * 0.5
+ }
+ PropertyChanges {
+ target: pushToTalk
+ opacity: 0.0
+ enabled: false
+ }
+ },
+ State {
+ name: "MicrophoneOff"
+ when: speechChromeController.chromeState == SpeechChromeController.MicrophoneOff
+ PropertyChanges {
+ target: chromeBarImage
+ opacity: 0.0
+ x: 0
+ }
+ PropertyChanges {
+ target: pushToTalk
+ opacity: 1.0
+ enabled: true
+ }
+ }
+ ]
+}
diff --git a/homescreen/qml/images/SpeechChrome/bar.png b/homescreen/qml/images/SpeechChrome/bar.png
new file mode 100644
index 0000000..caabde1
--- /dev/null
+++ b/homescreen/qml/images/SpeechChrome/bar.png
Binary files differ
diff --git a/homescreen/qml/images/SpeechChrome/push_to_talk.png b/homescreen/qml/images/SpeechChrome/push_to_talk.png
new file mode 100644
index 0000000..f45218e
--- /dev/null
+++ b/homescreen/qml/images/SpeechChrome/push_to_talk.png
Binary files differ
diff --git a/homescreen/qml/images/SpeechChrome/speechchrome.qrc b/homescreen/qml/images/SpeechChrome/speechchrome.qrc
new file mode 100644
index 0000000..8ab1472
--- /dev/null
+++ b/homescreen/qml/images/SpeechChrome/speechchrome.qrc
@@ -0,0 +1,6 @@
+<RCC>
+ <qresource prefix="/images/SpeechChrome">
+ <file>bar.png</file>
+ <file>push_to_talk.png</file>
+ </qresource>
+</RCC>
diff --git a/homescreen/qml/main.qml b/homescreen/qml/main.qml
index 7d40276..d74fd1f 100644
--- a/homescreen/qml/main.qml
+++ b/homescreen/qml/main.qml
@@ -90,6 +90,13 @@ Window {
}
}
+ SpeechChrome {
+ anchors.left: parent.left
+ anchors.right: parent.right
+ anchors.bottom: parent.bottom
+ height: 215
+ }
+
Connections {
target: homescreenHandler
onShowInformation: {
diff --git a/homescreen/qml/qml.qrc b/homescreen/qml/qml.qrc
index e60ea63..d901481 100644
--- a/homescreen/qml/qml.qrc
+++ b/homescreen/qml/qml.qrc
@@ -10,5 +10,6 @@
<file>StatusArea.qml</file>
<file>TopArea.qml</file>
<file>IconItem.qml</file>
+ <file>SpeechChrome.qml</file>
</qresource>
</RCC>
diff --git a/homescreen/src/aglsocketwrapper.cpp b/homescreen/src/aglsocketwrapper.cpp
new file mode 100755
index 0000000..8352660
--- /dev/null
+++ b/homescreen/src/aglsocketwrapper.cpp
@@ -0,0 +1,90 @@
+#include "aglsocketwrapper.h"
+#include "constants.h"
+
+#include <QWebSocket>
+#include <QUuid>
+#include <QJsonArray>
+#include <QJsonObject>
+#include <QJsonDocument>
+
+#include <QDebug>
+
+namespace {
+enum MessageTypes {
+ Call = 2,
+ Success = 3,
+ Error = 4,
+ Event = 5
+};
+}
+
+AglSocketWrapper::AglSocketWrapper(QObject *parent) :
+ QObject(parent)
+ , m_socket(new QWebSocket(QString(), QWebSocketProtocol::VersionLatest, this))
+{
+ connect(m_socket, &QWebSocket::connected, this, &AglSocketWrapper::connected);
+ connect(m_socket, &QWebSocket::disconnected, this, &AglSocketWrapper::disconnected);
+ connect(m_socket, QOverload<QAbstractSocket::SocketError>::of(&QWebSocket::error),
+ [](QAbstractSocket::SocketError error) -> void {
+ qWarning() << "AglSocketWrapper internal socket error" << error;
+ });
+ connect(m_socket, &QWebSocket::textMessageReceived,
+ this, [this](const QString &msg) -> void {
+ const QJsonDocument doc = QJsonDocument::fromJson(msg.toUtf8());
+ if (doc.isArray()) {
+ const QJsonArray msgArray = doc.array();
+ if (msgArray.count() >= 3) {
+ const int msgType = msgArray.at(0).toInt();
+ switch (msgType) {
+ case Success:
+ case Error: {
+ auto callbackIt = m_callbacks.find( msgArray.at(1).toString());
+ if (callbackIt != m_callbacks.constEnd()) {
+ (*callbackIt)(msgType == Success, msgArray.at(2));
+ m_callbacks.erase(callbackIt);
+ }
+ }
+ break;
+ case Event: {
+ const QJsonObject eventObj = msgArray.at(2).toObject();
+ emit eventReceived(msgArray.at(1).toString(), eventObj.value(vshl::DATA_TAG));
+ }
+ break;
+ default:
+ break;
+ }
+ return;
+ }
+ }
+ qWarning() << "Unsupported message format:" << msg;
+ });
+}
+
+void AglSocketWrapper::open(const QUrl &url)
+{
+ m_socket->open(url);
+}
+
+void AglSocketWrapper::close()
+{
+ m_socket->close();
+}
+
+void AglSocketWrapper::apiCall(const QString &api, const QString &verb, const QJsonValue &args,
+ AglSocketWrapper::ApiCallback callback)
+{
+ const QString id = QUuid::createUuid().toString();
+ if (callback)
+ m_callbacks.insert(id, callback);
+
+ QJsonArray callData;
+ callData.append(Call);
+ callData.append(id);
+ callData.append(api + QLatin1String("/") + verb);
+ callData.append(args);
+
+ const QString msg = QLatin1String(QJsonDocument(callData).toJson(QJsonDocument::Compact));
+ m_socket->sendTextMessage(msg);
+
+ qDebug() << Q_FUNC_INFO << "Data sent:" << msg;
+}
diff --git a/homescreen/src/aglsocketwrapper.h b/homescreen/src/aglsocketwrapper.h
new file mode 100755
index 0000000..4807cd5
--- /dev/null
+++ b/homescreen/src/aglsocketwrapper.h
@@ -0,0 +1,35 @@
+#ifndef AGLSOCKETWRAPPER_H
+#define AGLSOCKETWRAPPER_H
+
+#include <QUrl>
+#include <QMap>
+#include <QObject>
+#include <QJsonValue>
+
+#include <functional>
+
+class QWebSocket;
+class AglSocketWrapper : public QObject
+{
+ Q_OBJECT
+public:
+ explicit AglSocketWrapper(QObject *parent = nullptr);
+
+ void open(const QUrl &url);
+ void close();
+
+ using ApiCallback = std::function<void(bool, const QJsonValue&)>;
+ void apiCall(const QString &api, const QString &verb, const QJsonValue &args = QJsonValue(),
+ ApiCallback callback = nullptr);
+
+signals:
+ void connected();
+ void disconnected();
+ void eventReceived(const QString &eventName, const QJsonValue &data);
+
+private:
+ QWebSocket *m_socket;
+ QMap<QString, ApiCallback> m_callbacks;
+};
+
+#endif // AGLSOCKETWRAPPER_H
diff --git a/homescreen/src/chromecontroller.cpp b/homescreen/src/chromecontroller.cpp
new file mode 100644
index 0000000..7994fa0
--- /dev/null
+++ b/homescreen/src/chromecontroller.cpp
@@ -0,0 +1,137 @@
+#include "chromecontroller.h"
+#include "aglsocketwrapper.h"
+#include "constants.h"
+
+#include <QTimer>
+#include <QDebug>
+#include <QJsonDocument>
+
+ChromeController::ChromeController(const QUrl &bindingUrl, QObject *parent) :
+ QObject(parent)
+ , m_aglSocket(new AglSocketWrapper(this))
+{
+ //Alexa voice agent subscription----------------------------------------------------------------
+ {
+ connect(m_aglSocket, &AglSocketWrapper::connected,
+ this, [this]() -> void {
+ m_aglSocket->apiCall(vshl::API, vshl::VOICE_AGENT_ENUMERATION_VERB, QJsonValue(),
+ [this](bool result, const QJsonValue &data) -> void {
+ if (!result) {
+ qWarning() << "Failed to enumerate voice agents";
+ return;
+ }
+
+ QJsonObject dataObj = data.toObject();
+ auto objIt = dataObj.find(vshl::RESPONSE_TAG);
+ if (objIt == dataObj.constEnd()) {
+ qWarning() << "Voice agent enumeration response tag missing."
+ << dataObj;
+ return;
+ }
+
+ dataObj = objIt.value().toObject();
+ objIt = dataObj.find(vshl::AGENTS_TAG);
+ if (objIt == dataObj.constEnd()) {
+ qWarning() << "Voice agent enumeration agents tag missing."
+ << dataObj;
+ return;
+ }
+
+ const QJsonArray agents = objIt.value().toArray();
+ QString alexaAgentId;
+ for (const QJsonValue &agent : agents) {
+ const QJsonObject agentObj = agent.toObject();
+ auto agentIt = agentObj.find(vshl::NAME_TAG);
+ if (agentIt == agentObj.constEnd())
+ continue;
+ const QString agentName = agentIt.value().toString();
+ agentIt = agentObj.find(vshl::ID_TAG);
+ if (agentIt == agentObj.constEnd())
+ continue;
+ if (agentName.compare(vshl::ALEXA_AGENT_NAME) == 0) {
+ alexaAgentId = agentIt.value().toString();
+ break;
+ }
+ }
+ if (alexaAgentId.isEmpty()) {
+ qWarning() << "Alexa voice agent not found";
+ return;
+ }
+
+ //Voice agent subscription------------------------------------------------------
+ {
+ m_voiceAgentId = alexaAgentId;
+ const QJsonObject args {
+ { vshl::VOICE_AGENT_ID_ARG, alexaAgentId },
+ { vshl::VOICE_AGENT_EVENTS_ARG, vshl::ALEXA_EVENTS_ARRAY }
+ };
+ m_aglSocket->apiCall(vshl::API, vshl::SUBSCRIBE_VERB, args,
+ [](bool result, const QJsonValue &data) -> void {
+ qDebug() << (vshl::API + QLatin1String(":") + vshl::SUBSCRIBE_VERB)
+ << "result: " << result << " val: " << data;
+ });
+ }
+ //------------------------------------------------------------------------------
+ });
+ });
+ }
+ //----------------------------------------------------------------------------------------------<
+
+ //Socket connection management------------------------------------------------------------------
+ {
+ auto connectToBinding = [bindingUrl, this]() -> void {
+ m_aglSocket->open(bindingUrl);
+ qDebug() << "Connecting to:" << bindingUrl;
+ };
+ connect(m_aglSocket, &AglSocketWrapper::disconnected, this, [connectToBinding]() -> void {
+ QTimer::singleShot(2500, connectToBinding);
+ });
+ connectToBinding();
+ }
+ //----------------------------------------------------------------------------------------------
+
+ //Speech chrome state change event handling-----------------------------------------------------
+ {
+ connect(m_aglSocket, &AglSocketWrapper::eventReceived,
+ this, [this](const QString &eventName, const QJsonValue &data) -> void {
+ if (eventName.compare(vshl::VOICE_DIALOG_STATE_EVENT + m_voiceAgentId) == 0) {
+ const QJsonObject dataObj = QJsonDocument::fromJson(data.toString().toUtf8()).object();
+ auto objIt = dataObj.find(vshl::STATE_TAG);
+ if (objIt == dataObj.constEnd()) {
+ qWarning() << "Voice dialog state event state missing.";
+ return;
+ }
+ const QString stateStr = objIt.value().toString();
+ if (stateStr.compare(vshl::VOICE_DIALOG_IDLE) == 0) {
+ setChromeState(Idle);
+ } else if (stateStr.compare(vshl::VOICE_DIALOG_LISTENING) == 0) {
+ setChromeState(Listening);
+ } else if (stateStr.compare(vshl::VOICE_DIALOG_THINKING) == 0) {
+ setChromeState(Thinking);
+ } else if (stateStr.compare(vshl::VOICE_DIALOG_SPEAKING) == 0) {
+ setChromeState(Speaking);
+ } else if (stateStr.compare(vshl::VOICE_DIALOG_MICROPHONEOFF) == 0) {
+ setChromeState(MicrophoneOff);
+ }
+ }
+ });
+ }
+ //----------------------------------------------------------------------------------------------
+}
+
+void ChromeController::pushToTalk()
+{
+ m_aglSocket->apiCall(vshl::API, vshl::TAP_TO_TALK_VERB, QJsonValue(),
+ [](bool result, const QJsonValue &data) -> void {
+ qDebug() << (vshl::API + QLatin1String(":") + vshl::TAP_TO_TALK_VERB)
+ << "result: " << result << " val: " << data;
+ });
+}
+
+void ChromeController::setChromeState(ChromeController::ChromeState state)
+{
+ if (m_chromeState != state) {
+ m_chromeState = state;
+ emit chromeStateChanged();
+ }
+}
diff --git a/homescreen/src/chromecontroller.h b/homescreen/src/chromecontroller.h
new file mode 100644
index 0000000..27c26cb
--- /dev/null
+++ b/homescreen/src/chromecontroller.h
@@ -0,0 +1,38 @@
+#pragma once
+
+#include <QObject>
+#include <QUrl>
+
+class AglSocketWrapper;
+class ChromeController : public QObject
+{
+ Q_OBJECT
+
+ Q_PROPERTY(int chromeState READ chromeState NOTIFY chromeStateChanged)
+
+public:
+ enum ChromeState {
+ Idle = 0,
+ Listening,
+ Thinking,
+ Speaking,
+ MicrophoneOff
+ };
+ Q_ENUM(ChromeState)
+
+ explicit ChromeController(const QUrl &bindingUrl, QObject *parent = nullptr);
+ int chromeState() const { return m_chromeState; }
+
+public slots:
+ void pushToTalk();
+
+signals:
+ void chromeStateChanged();
+
+private:
+ void setChromeState(ChromeState state);
+
+ AglSocketWrapper *m_aglSocket;
+ QString m_voiceAgentId;
+ ChromeState m_chromeState = Idle;
+};
diff --git a/homescreen/src/constants.h b/homescreen/src/constants.h
new file mode 100644
index 0000000..38de9b9
--- /dev/null
+++ b/homescreen/src/constants.h
@@ -0,0 +1,40 @@
+#ifndef CONSTANTS_H
+#define CONSTANTS_H
+
+#include <QString>
+#include <QJsonArray>
+#include <QJsonObject>
+
+namespace vshl {
+const QString API = QLatin1String("vshl-core");
+const QString VOICE_AGENT_ENUMERATION_VERB = QLatin1String("enumerateVoiceAgents");
+const QString SUBSCRIBE_VERB = QLatin1String("subscribe");
+const QString TAP_TO_TALK_VERB = QLatin1String("startListening");
+
+const QString ALEXA_AGENT_NAME = QLatin1String("Alexa");
+const QJsonArray ALEXA_EVENTS_ARRAY = {
+ QLatin1String("voice_authstate_event"),
+ QLatin1String("voice_dialogstate_event"),
+ QLatin1String("voice_connectionstate_event")
+};
+
+const QString DATA_TAG = QLatin1String("data");
+const QString RESPONSE_TAG = QLatin1String("response");
+const QString AGENTS_TAG = QLatin1String("agents");
+const QString NAME_TAG = QLatin1String("name");
+const QString ID_TAG = QLatin1String("id");
+const QString STATE_TAG = QLatin1String("state");
+
+const QString VOICE_AGENT_ID_ARG = QLatin1String("va_id");
+const QString VOICE_AGENT_EVENTS_ARG = QLatin1String("events");
+const QString VOICE_AGENT_ACTIONS_ARG = QLatin1String("actions");
+
+const QString VOICE_DIALOG_STATE_EVENT = QLatin1String("vshl-core/voice_dialogstate_event#");
+const QString VOICE_DIALOG_IDLE = QLatin1String("IDLE");
+const QString VOICE_DIALOG_LISTENING = QLatin1String("LISTENING");
+const QString VOICE_DIALOG_THINKING = QLatin1String("THINKING");
+const QString VOICE_DIALOG_SPEAKING = QLatin1String("SPEAKING");
+const QString VOICE_DIALOG_MICROPHONEOFF = QLatin1String("MICROPHONEOFF");
+}
+
+#endif // CONSTANTS_H
diff --git a/homescreen/src/main.cpp b/homescreen/src/main.cpp
index 5f283fb..5c819f9 100644
--- a/homescreen/src/main.cpp
+++ b/homescreen/src/main.cpp
@@ -32,6 +32,7 @@
#include "mastervolume.h"
#include "homescreenhandler.h"
#include "hmi-debug.h"
+#include "chromecontroller.h"
// XXX: We want this DBus connection to be shared across the different
// QML objects, is there another way to do this, a nice way, perhaps?
@@ -91,6 +92,8 @@ int main(int argc, char *argv[])
// qmlRegisterType<ApplicationLauncher>("HomeScreen", 1, 0, "ApplicationLauncher");
qmlRegisterType<StatusBarModel>("HomeScreen", 1, 0, "StatusBarModel");
qmlRegisterType<MasterVolume>("MasterVolume", 1, 0, "MasterVolume");
+ qmlRegisterUncreatableType<ChromeController>("SpeechChrome", 1, 0, "SpeechChromeController",
+ QLatin1String("SpeechChromeController is uncreatable."));
ApplicationLauncher *launcher = new ApplicationLauncher();
QLibWindowmanager* layoutHandler = new QLibWindowmanager();
@@ -140,6 +143,7 @@ int main(int argc, char *argv[])
engine.rootContext()->setContextProperty("launcher", launcher);
engine.rootContext()->setContextProperty("weather", new Weather(bindingAddress));
engine.rootContext()->setContextProperty("bluetooth", new Bluetooth(bindingAddress, engine.rootContext()));
+ engine.rootContext()->setContextProperty("speechChromeController", new ChromeController(bindingAddress, &engine));
engine.rootContext()->setContextProperty("screenInfo", &screenInfo);
engine.load(QUrl(QStringLiteral("qrc:/main.qml")));
diff --git a/package/config.xml b/package/config.xml
index a15baee..56d9b40 100644
--- a/package/config.xml
+++ b/package/config.xml
@@ -13,6 +13,7 @@
<param name="Bluetooth-Manager" value="ws" />
<param name="windowmanager" value="ws" />
<param name="ahl-4a" value="ws" />
+ <param name="vshl-core" value="ws" />
</feature>
<feature name="urn:AGL:widget:required-permission">
<param name="urn:AGL:permission::public:no-htdocs" value="required" />