A dialog flow app

Dialogflow provided a easy to configured way to integrate the voice interaction with your application. By using their sample code, it is easy to implement and deploy to multiple different platform. But it also have problems such that wrong information matching, can not define intent/entity on the go, privacy issue, and etc. This is a course assignment done solely by me. More details in below:

This one requires the creation of dialogflow project on their official site by creating intent and entities. And in the local side, just create a locale server and enable webhook on the dialogflow website.

Key terms:

agent This is a nearly global variable, which contains the information user could get and will send to the server. It needs to be initialized in the beginning of the setting:

app.post('/', express.json(), (req, res) => {
  const agent = new WebhookClient({ request: req, response: res })

  let intentMap = new Map() //and using an intentMap to match the intent and function to execute. like below: 
  intentMap.set('Default Welcome Intent', function () => {})//Default Welcome Intent is the name
  agent.handleRequest(intentMap)
}

agent.add(message) method to add return information.

Caution: In this method, multiple times of calling will concat information instead of random choosing and replying.

agent.parameters.paraName get access to the detected entity (named as paraName).

Caution: paraName might return a string that is totally not in your defined entity list. This one provides convenience but also loss the precise. So, be sure to have a good program that is able to handle those stuff correctly

In my case, there is a entity list defined as [cloth, dress, hat, …] (no this) and dialogflow detect the parameter with this as the value.

agent.query the entire sentence user said

There are other advanced stuff like addContext(), etc… Those four are enough to build an application.

A complete simple sample:

app.post('/', express.json(), (req, res) => {
  const agent = new WebhookClient({ request: req, response: res })

  let intentMap = new Map() //and using an intentMap to match the intent and function to execute. like below: 
  intentMap.set('Default Welcome Intent', function () => {
                agent.add("aaa" + ":user words:" + agent.query)
                })//Default Welcome Intent is the name
  agent.handleRequest(intentMap)
}
app.listen(process.env.PORT || 8080)

Discussion

Pros:

1. Super easy to use, a two hour learning is enough to build an application
2. Have a lot of sample
3. Have a more powerful training platform.
4. Can easily integrate into platforms and deliver to user.

Cons:

1. Dialogflow provided a easy to configured way to integrate the voice interaction with most applications. But their compatibility focuses on google platform. Platform like Expo gets to eject to integrate with it.
2. Dialogflow will time out after 5 seconds, which means the server need to run everything super fast. But this is hard in some cases, when network are not good, or in some network limited area.
3. Entity extraction might match wrong and really broad keyword that backend do not know how to prevent, like match this with dress, personally thinking Nlp.js is a better tool, refer to Nlp.js in Expo usage to see more about implementing on it.
4. Possible privacy issue.
5. It is impossible to define intent/entity based on user needs. Even though one model could work for majority people, personalization could make user feel much more comfortable to certain people. And this could eliminate training phrases as well.

Entire source code:

const express = require('express')
const { WebhookClient } = require('dialogflow-fulfillment')
const app = express()
const fetch = require('node-fetch')
const base64 = require('base-64')

let username = "";
let password = "";
let token = "";
let products;
let filtered;

USE_LOCAL_ENDPOINT = false;
// set this flag to true if you want to use a local endpoint
// set this flag to false if you want to use the online endpoint
ENDPOINT_URL = ""
if (USE_LOCAL_ENDPOINT) {
  ENDPOINT_URL = "http://127.0.0.1:5000"
} else {
  ENDPOINT_URL = "XXX"
}

fetch(ENDPOINT_URL + '/products').then(res => res.json()).then(data => {
  for (let i of data.products) {
    fetch(ENDPOINT_URL + '/products/' + i.id + "/tags").then(res => res.json()).then(data1 => {
      i["tags"] = data1.tags;
    })
  }
  products = data
})



async function navigation(back, dialogflow, page) {
  let request;
  if (page === null) {
    request = {
      method: 'PUT',
      headers: {
        'Content-Type': 'application/json',
        'x-access-token': token
      },
      body: JSON.stringify({
        "back": back,
        "dialogflowUpdated": dialogflow
      })
    }
  } else {
    request = {
      method: 'PUT',
      headers: {
        'Content-Type': 'application/json',
        'x-access-token': token
      },
      body: JSON.stringify({
        "back": back,
        "dialogflowUpdated": dialogflow,
        "page": "/" + page
      })
    }
  }


  const serverReturn1 = await fetch(ENDPOINT_URL + '/application', request)
  const serverResponse1 = await serverReturn1.json()
}

async function getToken(message) {
  let request = {
    method: 'GET',
    headers: {
      'Content-Type': 'application/json',
      'Authorization': 'Basic ' + base64.encode(username + ':' + password)
    },
    redirect: 'follow'
  }

  const serverReturn = await fetch(ENDPOINT_URL + '/login', request)
  if (serverReturn.ok) {
    const serverResponse = await serverReturn.json()
    token = serverResponse.token

    request = {
      method: 'PUT',
      headers: {
        'Content-Type': 'application/json',
        'x-access-token': token
      },
      body: JSON.stringify({
        "back": false,
        "dialogflowUpdated": true,
        "page": "/" + username
      })
    }

    const serverReturn2 = await fetch(ENDPOINT_URL + '/application/messages', {
      method: 'DELETE',
      headers: {
        'x-access-token': token
      }
    })
    serverReturn2.json()

    addUserMessage(message);


    const serverReturn1 = await fetch(ENDPOINT_URL + '/application', request)
    const serverResponse1 = await serverReturn1.json()

    //console.log(products)

    return token;
  } else {
    username = "";
    password = "";
    return "fail";
  }
}

async function addUserMessage(user) {
  const serverReturn2 = await fetch(ENDPOINT_URL + '/application/messages', {
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      'x-access-token': token
    },
    body: JSON.stringify({
      "isUser": true,
      "date": new Date().toISOString(),
      "text": user
    })
  })
  await serverReturn2.json()
}


async function addAgentMessage(agent) {
  const serverReturn2 = await fetch(ENDPOINT_URL + '/application/messages', {
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      'x-access-token': token
    },
    body: JSON.stringify({
      "isUser": false,
      "date": new Date().toISOString(),
      "text": agent
    })
  })
  await serverReturn2.json()

}

app.get('/', (req, res) => res.send('online'))
app.post('/', express.json(), (req, res) => {
  const agent = new WebhookClient({ request: req, response: res })

  function welcome() {
    agent.add('Webhook works!')
    console.log(ENDPOINT_URL)
  }

  async function login() {
    //console.log(products.products[0].tags);

    if (token.localeCompare("") !== 0) {
      await addUserMessage(agent.query);
      agent.add("You have logged in as " + username);
      await addAgentMessage("You have logged in as " + username);
    }
    // You need to set this from `username` entity that you declare in DialogFlow
    username = agent.parameters.username;
    // You need to set this from password entity that you declare in DialogFlow
    password = agent.parameters.password;
    await getToken(agent.query);

    if (token.localeCompare("") === 0) {
      agent.add("login failed, try again with login");
    } else {
      agent.add("Successfully logged in as " + username);
      addAgentMessage("Successfully logged in as " + username);
    }
  }

  async function loginStatus() {
    if (token.localeCompare("") === 0) {
      if (Math.random() > 0.5)
        agent.add("You are not logged in. ");
      else agent.add("Please log in at first");
    } else {
      await addUserMessage(agent.query);
      if (Math.random() < 0.3) {
        agent.add("You are " + username);
        await addAgentMessage("You are " + username);
      }
      else if (Math.random() < 0.6) {
        agent.add("You username is " + username);
        await addAgentMessage("You username is " + username);
      } else {
        agent.add("Hello, " + username);
        await addAgentMessage("Hello, " + username);

      }
    }
  }

  async function queryCategories() {
    if (token.localeCompare("") === 0) {
      //ensure user is logged in
      agent.add("Invalid visit, please login first");
      return;
    }
    //console.log(agent.parameters.category);
    await addUserMessage(agent.query);
    await navigation(false, true, username + "/" + agent.parameters.category);
    filtered = [];
    let k = [];
    for (let i of products.products) {
      if (i.category.localeCompare(agent.parameters.category) === 0) {
        filtered.push(i);
        k.push(i.name)
      }
    }
    agent.add("I found " + k.length === 0 ? "nothing" : k.toString() + " belongs to " + agent.parameters.category)
    addAgentMessage("I found " + k.length === 0 ? "nothing" : k.toString() + " belongs to " + agent.parameters.category);

  }

  async function filterTags() {
    if (token.localeCompare("") === 0) {
      //ensure user is logged in
      agent.add("Invalid visit, please login first");
      return;
    }
    await addUserMessage(agent.query)

    let serve = await fetch(ENDPOINT_URL + '/application/tags/' + agent.parameters.tags, {
      method: 'POST',
      headers: {
        'x-access-token': token
      }
    })
    await serve.json()
    if (serve.ok) {
      let k = " ";
      for (let i of filtered) {
        if (i.tags.indexOf(agent.parameters.tags) !== -1) {
          k += i.name + ", ";
        }
      }
      agent.add("Successfully add tag " + agent.parameters.tags + ", filtered items: " + k)//need filter and return something here
      addAgentMessage("Successfully add tag " + agent.parameters.tags + ", filtered items: " + k)
    } else {
      agent.add("Unknown problem happened");
      addAgentMessage("Unknown problem happened")
    }
  }

  async function clearTags() {
    if (token.localeCompare("") === 0) {
      //ensure user is logged in
      agent.add("Invalid visit, please login first");
      return;
    }
    await addUserMessage(agent.query)
    let serve = await fetch(ENDPOINT_URL + '/application/tags', {
      method: 'DELETE',
      headers: {
        'x-access-token': token
      }
    })
    await serve.json()
    if (serve.ok) {
      let k = " ";
      for (let i of filtered) {
        k += i.name + ", ";
      }
      agent.add("Successfully cleared all tags filters, current items are " + k)
      addAgentMessage("Successfully cleared all tags filters, current items are " + k)
    } else {
      agent.add("Unknown problem happened");
      addAgentMessage("Unknown problem happened")
    }
  }

  async function queryCategoryTag() {
    if (token.localeCompare("") === 0) {
      //ensure user is logged in
      agent.add("Invalid visit, please login first");
      return;
    }
    await addUserMessage(agent.query)
    let serve = await fetch(ENDPOINT_URL + '/application/tags', {
      method: 'DELETE',
      headers: {
        'x-access-token': token
      }
    })
    await serve.json()

    if (serve.ok) {

      await navigation(false, true, username + "/" + agent.parameters.category);
      filtered = [];
      for (let i of products.products) {
        if (i.category.localeCompare(agent.parameters.category) === 0) {
          filtered.push(i);
        }
      }

      let serve1 = await fetch(ENDPOINT_URL + '/application/tags/' + agent.parameters.tags, {
        method: 'POST',
        headers: {
          'x-access-token': token
        }
      })
      await serve1.json()
      if (serve1.ok) {
        let k = " ";
        for (let i of filtered) {
          if (i.tags.indexOf(agent.parameters.tags) !== -1) {
            k += i.name + ", ";
          }
        }
        agent.add("Successfully add tag " + agent.parameters.tags + ", filtered items: " + k)//need filter and return something here
        addAgentMessage("Successfully add tag " + agent.parameters.tags + ", filtered items: " + k)
      } else {
        agent.add("Unknown problem happened");
        addAgentMessage("Unknown problem happened")
      }
    } else {
      agent.add("Unknown problem happened");
      addAgentMessage("Unknown problem happened")
    }
  }

  async function goBack() {
    if (token.localeCompare("") === 0) {
      //ensure user is logged in
      agent.add("Invalid visit, please login first");
      return;
    }
    await addUserMessage(agent.query)
    await navigation(true, true, null);
    agent.add("Go back successfully");
    await addAgentMessage("Go back successfully")
  }

  async function productRating() {
    if (token.localeCompare("") === 0) {
      //ensure user is logged in
      agent.add("Invalid visit, please login first");
      return;
    }
    await addUserMessage(agent.query);

    let id;
    for (let i of products.products) {
      if (agent.parameters.product.indexOf(i.name) !== -1) {
        id = i.id;
        break;
      }
    }

    let serve = await fetch(ENDPOINT_URL + '/products/' + id + "/reviews")
    let reviews = await serve.json()


    if (reviews.reviews.length !== 0) {
      //review exist
      let reviewText = [];
      let rating = 0;

      for (let i of reviews.reviews) {
        rating += i.stars;
        if (i.text.localeCompare("<Product Review Text>") !== 0) {
          reviewText.push("Title: " + i.title)
          reviewText.push("Text: " + i.text)
        }
      }

      agent.add("This items has the average rating of " + (rating / reviews.reviews.length).toFixed(2) + ". It also contains comments of the following: " + reviewText.toString())
      addAgentMessage("This items has the average rating of " + (rating / reviews.reviews.length).toFixed(2) + ". It also contains comments of the following: " + reviewText.toString())
    } else {
      agent.add("Review do not exist")
      addAgentMessage("Review do not exist")
    }

  }

  async function viewProduct() {
    if (token.localeCompare("") === 0) {
      //ensure user is logged in
      agent.add("Invalid visit, please login first");
      return;
    }
    await addUserMessage(agent.query);


    let id;
    let catego;
    for (let i of products.products) {
      if (agent.parameters.product.toLocaleLowerCase().indexOf(i.name.toLocaleLowerCase()) !== -1) {
        id = i.id;
        catego = i.category;
        break;
      }
    }

    let serve = await fetch(ENDPOINT_URL + '/products/' + id + "/reviews")
    let reviews = await serve.json()

    navigation(false, true, username + "/" + catego + "/products/" + id)
    agent.add("Navigate you to " + agent.parameters.product + "page")
    addAgentMessage("Navigate you to " + agent.parameters.product + "page")

  }

  async function addProduct() {
    if (token.localeCompare("") === 0) {
      //ensure user is logged in
      agent.add("Invalid visit, please login first");
      return;
    }

    await addUserMessage(agent.query);

    let id = null;

    if (agent.parameters.product.localeCompare("") === 0 || agent.parameters.product.toLocaleLowerCase().indexOf("product") !== -1) {
      //get info from page
      const serverReturn2 = await fetch(ENDPOINT_URL + '/application/', {
        method: 'GET',
        headers: {
          'x-access-token': token
        }
      })

      //page field
      let result = await serverReturn2.json()
      result = result.page.split("/");
      if (result.length > 2) {
        if (result[result.length - 2].localeCompare("products") === 0) {
          //normal add
          id = parseInt(result[result.length - 1]);

        } else {
          agent.add("You are not in a product page");
          addAgentMessage("You are not in a product page");
          return;
        }
      } else {
        agent.add("You are not in a product page");
        addAgentMessage("You are not in a product page");
        return;
      }

    } else {
      let toFind = agent.parameters.product.toLocaleLowerCase();
      for (let i of products.products) {
        let tocompare = i.name.toLocaleLowerCase();
        if (toFind.indexOf(tocompare) !== -1) {
          id = i.id;
          break;
        }
      }
    }

    if (id === null) {
      agent.add("Product name has problem");
      addAgentMessage("Product name has problem");
      return;
    }

    //adding

    let serve = await fetch(ENDPOINT_URL + '/application/products/' + id, {
      method: 'POST',
      headers: {
        'Content-Type': 'application/json',
        'x-access-token': token
      }
    })

    //page field
    await serve.json()
    if (serve.ok) {
      agent.add("Successfully added 1 of " + agent.parameters.product + " to your cart")
      addAgentMessage("Successfully added 1 of " + agent.parameters.product + " to your cart")
    } else {
      agent.add("Failed with server problem");
      addAgentMessage("Failed with server problem");
    }
  }

  async function cleanCart() {
    if (token.localeCompare("") === 0) {
      //ensure user is logged in
      agent.add("Invalid visit, please login first");
      return;
    }
    await addUserMessage(agent.query);


    let serve = await fetch(ENDPOINT_URL + '/application/products/', {
      method: 'DELETE',
      headers: {
        'Content-Type': 'application/json',
        'x-access-token': token
      }
    })

    await serve.json()
    if (serve.ok) {
      agent.add("Successfully cleaned your cart")
      addAgentMessage("Successfully cleaned your cart")
    } else {
      agent.add("Failed with server problem");
      addAgentMessage("Failed with server problem");
    }
  }

  async function removeProduct() {
    if (token.localeCompare("") === 0) {
      //ensure user is logged in
      agent.add("Invalid visit, please login first");
      return;
    }
    await addUserMessage(agent.query);

    let id = null;
    for (let i of products.products) {
      if (agent.parameters.product.toLocaleLowerCase().indexOf(i.name.toLocaleLowerCase()) !== -1) {
        id = i.id;
        break;
      }
    }

    if (id === null) {
      agent.add("Product name has problem");
      addAgentMessage("Product name has problem");
      return;
    }

    let serve = await fetch(ENDPOINT_URL + '/application/products/' + id, {
      method: 'DELETE',
      headers: {
        'Content-Type': 'application/json',
        'x-access-token': token
      }
    })

    await serve.json()
    if (serve.ok) {
      agent.add("Successfully removed 1 of " + agent.parameters.product + " from your cart")
      addAgentMessage("Successfully removed 1 of " + agent.parameters.product + " from your cart")
    } else {
      agent.add("Failed with server problem");
      addAgentMessage("Failed with server problem");
    }
  }

  async function navigatePage() {
    if (token.localeCompare("") === 0) {
      //ensure user is logged in
      agent.add("Invalid visit, please login first");
      return;
    }
    await addUserMessage(agent.query);

    let findContent = agent.parameters.pages.toLocaleLowerCase();

    if (findContent.indexOf("home") !== -1) {
      navigation(false, true, username)
      agent.add("Navigating you to home...");
      addAgentMessage("Navigating you to home...");
      return;
    }

    
    if (findContent.indexOf("review") !== -1) {
      navigation(false, true, username + "/cart-review")
      agent.add("Navigating you to cart-review...");
      addAgentMessage("Navigating you to cart-review...");
      return;
    }

    if (findContent.indexOf("confirm") !== -1) {
      const serverReturn2 = await fetch(ENDPOINT_URL + '/application/', {
        method: 'GET',
        headers: {
          'x-access-token': token
        }
      })

      //page field
      let result = await serverReturn2.json()
      result = result.page.split("/");
      if (result.length > 1) {
        if (result[result.length - 1].localeCompare("cart-review") === 0) {

          navigation(false, true, username + "/cart-confirmed")
          agent.add("Navigating you to cart-confirmed...");
          addAgentMessage("Navigating you to cart-confirmed...");
          return;
        } else {
          agent.add("You are not in a cart review page");
          addAgentMessage("You are not in a cart review page");
          return;
        }
      } else {
        agent.add("You are not in a cart review page");
        addAgentMessage("You are not in a cart review page");
        return;
      }
    }

    if (findContent.indexOf("cart") !== -1) {
      navigation(false, true, username + "/cart")
      agent.add("Navigating you to cart...");
      addAgentMessage("Navigating you to cart...");
      return;
    }

  }

  async function cartInfo() {
    if (token.localeCompare("") === 0) {
      //ensure user is logged in
      agent.add("Invalid visit, please login first");
      return;
    }
    await addUserMessage(agent.query);


    let serve = await fetch(ENDPOINT_URL + '/application/products/', {
      method: 'GET',
      headers: {
        'Content-Type': 'application/json',
        'x-access-token': token
      }
    })

    let cart = await serve.json();
    cart = cart.products;
    navigation(false, true, username + "/cart");

    let findContent = agent.parameters.cartInformation.toLocaleLowerCase();

    if (findContent.indexOf("number") !== -1) {
      let k = 0;
      for (let i of cart) {
        k += i.count;
      }
      agent.add("You have " + k + " items in your cart");
      addAgentMessage("You have " + k + " items in your cart");
      return;
    }

    if (findContent.indexOf("cost") !== -1 || findContent.indexOf("price") !== -1) {
      let k = 0;
      for (let i of cart) {
        k += i.count * i.price;
      }
      agent.add("Total price is " + k.toFixed(2) + " in your cart");
      addAgentMessage("Total price is " + k.toFixed(2) + " in your cart");
      return;
    }

    if (findContent.indexOf("type") !== -1) {//assume to be category
      let k = [];
      for (let i of cart) {
        if (k.indexOf(i.category) === -1) {
          k.push(i.category);
        }
      }
      agent.add("You have items belong to the following categories " + k.length === 0 ? "nothing" : k.toString() + " in your cart");
      addAgentMessage("You have items belong to the following categories " + k.length === 0 ? "nothing" : k.toString() + " in your cart");
      return;
    }

    if (findContent.indexOf("item") !== -1) {//assume to be items name + count
      let k = [];
      for (let i of cart) {
        k.push(i.count + " of " + i.name);
      }
      agent.add("You have items " + k.length === 0 ? "nothing" : k.toString() + " in your cart");
      addAgentMessage("You have items " + k.length === 0 ? "nothing" : k.toString() + " in your cart");
      return;
    }

    //entering default, all info
    let toSend = "";
    let k = 0;
    for (let i of cart) {
      k += i.count;
    }
    toSend += "You have " + k + " items in your cart ";

    k = 0
    for (let i of cart) {
      k += i.count * i.price;
    }
    toSend += "Total price is " + k.toFixed(2) + " in your cart ";

    k = [];
    for (let i of cart) {
      if (k.indexOf(i.category) === -1) {
        k.push(i.category);
      }
    }
    toSend += "You have items belong to the following categories " + k.toString() + " in your cart";

    k = [];
    for (let i of cart) {
      k.push(i.count + " of " + i.name);
    }
    toSend += "You have items " + k.toString() + " in your cart";

    agent.add(toSend);
    addAgentMessage(toSend);

  }




  let intentMap = new Map()
  intentMap.set('Default Welcome Intent', welcome)
  // You will need to declare this `Login` content in DialogFlow to make this work

  intentMap.set('LoginStatus', loginStatus)
  intentMap.set('login', login)
  intentMap.set('queryCategories', queryCategories)
  intentMap.set('filterTags', filterTags)
  intentMap.set('clearTags', clearTags)
  intentMap.set('queryCategoryTag', queryCategoryTag)
  intentMap.set('goBack', goBack)
  intentMap.set('productRating', productRating)
  intentMap.set('viewProduct', viewProduct)
  intentMap.set('addProduct', addProduct)
  intentMap.set('cleanCart', cleanCart)
  intentMap.set('removeProduct', removeProduct)
  intentMap.set('removeProduct', removeProduct)
  intentMap.set('navigatePage', navigatePage)
  intentMap.set('cartInfo', cartInfo)



  agent.handleRequest(intentMap)
})

app.listen(process.env.PORT || 8080)