Localhost (#12)

* support .env files

* reorg services

* formatting

* move stuff out of storage

* avoid import *

* use azurite sim

* plug azurite

* error messages

* adding local.env

* start local

* dev by default

* updated docs

* docs

* docs

Author: pelikhan

.gitignore

@@ -14,4 +14,5 @@ pkg.zip
 tmp
 *.parameters.json
 debug.mjs
-pkgtmp
+pkgtmp
+.azurite

README.md

@@ -3,6 +3,8 @@
 This project contains a prototype development gateway implementation
 for the built-in DeviceScript cloud integration.
 
+The gateway can be run locally (no cloud dependencies) or deployed to Azure.
+
 -   [Read the documentation](https://microsoft.github.io/devicescript/developer/gateway)
 
 | :exclamation: This implementation is for prototyping only and not meant for production. |
@@ -12,12 +14,32 @@ for the built-in DeviceScript cloud integration.
 
 Make sure to follow the provisioning steps in the documentation before trying to run locally.
 
--   start a local instance using
+-   start azurite in a terminal
 
+```bash
+yarn azurite
 ```
+
+-   start a local instance using azurite
+
+```bash
 yarn dev
 ```
 
+-   after running head to http://127.0.0.1:7071/swagger/ or otherwise the live site
+-   Click Authorize
+-   Use user/password `devstoreaccount1:Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==`
+
+## Azure services
+
+Make sure to follow the provisioning steps in the documentation before trying to run locally.
+
+-   start a local instance using Azure services
+
+```
+yarn dev:azure
+```
+
 -   after running head to http://127.0.0.1:7071/swagger/ or otherwise the live site
 -   Click Authorize
 -   Use user/password from the `passwords` secret in the key vault

dev.mjs

@@ -1,5 +1,5 @@
 import dotenv from "dotenv"
-const out = dotenv.config()
-if (out.error)
-    throw out.error
+const azure = process.argv.includes("--azure")
+const out = dotenv.config({ path: azure ? "./.env" : "./local.env" })
+if (out.error) throw out.error
 import("./dist/src/index.js")

devicescript

@@ -0,0 +1,7 @@
+WEBSITE_SITE_NAME="local"
+WEBSITE_HOSTNAME=127.0.0.1:7071
+DEVS_LOCALHOST = "1"
+DEVS_STORAGE_CONNECTION_STRING = "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;TableEndpoint=http://127.0.0.1:10002/devstoreaccount1;"
+DEVS_PASSWORDS_SECRET="DEVS_LOCAL_USER_PASSWORD"
+DEVS_LOCAL_USER_PASSWORD = "devstoreaccount1:Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="
+DEVS_SWAGGER_URL="http://127.0.0.1:7071/swagger/"

local.env

@@ -17,8 +17,9 @@
     "build": "tsc",
     "watch": "tsc -w",
     "start": "node dist/src/index.js",
-    "local": "node dev.mjs",
-    "dev": "yarn build && yarn local",
+    "azurite": "azurite --location ./.azurite --debug ./.azurite/debug.log",
+    "dev": "yarn build && node dev.mjs --local",
+    "dev:azure": "yarn build && node dev.mjs --azure",
     "logs": "node infra/logs.mjs",
     "deploy": "node infra/zipdeploy.mjs"
   },
@@ -36,6 +37,7 @@
     "applicationinsights": "^2.5.0",
     "fastify": "^4.9.2",
     "http-errors": "^2.0.0",
+    "json5": "^2.2.3",
     "node-fetch": "^2.6.7",
     "openapi-types": "^12.1.0",
     "pino-pretty": "^9.1.1",
@@ -48,11 +50,12 @@
     "@types/node": "16.x",
     "@types/ws": "^8.5.3",
     "archiver": "^5.3.1",
+    "azurite": "^3.23.0",
     "dotenv": "^16.0.3",
     "libsodium-wrappers": "^0.7.11",
     "open": "^8.4.2",
     "typescript": "^4.0.0",
     "xmlbuilder2": "^3.0.2",
     "zx": "^7.2.0"
   }
-}
+}

package.json

@@ -1,17 +1,30 @@
 import { randomBytes } from "crypto"
 import { FastifyInstance, FastifyRequest } from "fastify"
-import * as storage from "./storage"
 import {
     checkString,
     displayName,
     sanitizeDeviceId,
+    selfUrl,
     throwStatus,
     tryParseJSON,
 } from "./util"
 import { DeviceId, DeviceInfo, FromDeviceMessage } from "./schema"
 import { wsskConnString } from "./wssk"
 import { fullDeviceId, pingDevice, pubToDevice } from "./devutil"
 import { fwdSockConnSettings } from "./fwdsock"
+import {
+    addMessageHook,
+    createDevice,
+    deleteDevice,
+    deleteMessageHook,
+    deviceStats,
+    getDevice,
+    getDeviceList,
+    getScript,
+    listMessageHooks,
+    stringifyMeta,
+    updateDevice,
+} from "./storage"
 
 const CONNECTED_TIMEOUT = 2 * 60 * 1000
 export const MAX_WSSK_SIZE = 230 // for to-device JSON and binary messages
@@ -29,12 +42,12 @@ function externalDevice(info: DeviceInfo) {
         deployedHash: info.deployedHash,
         lastAct: info.lastAct ? new Date(info.lastAct).toISOString() : "",
         meta: tryParseJSON(info.metaJSON),
-        stats: storage.deviceStats(info),
+        stats: deviceStats(info),
     }
 }
 
 async function singleDevice(id: DeviceId) {
-    const dev = await storage.getDevice(id)
+    const dev = await getDevice(id)
     if (!dev) throwStatus(404, "no such device")
     return externalDevice(dev)
 }
@@ -53,7 +66,7 @@ export async function getDeviceFromFullPath(req: FastifyRequest) {
     if (!(typeof part == "string" && /^\w{2,32}$/.test(part)))
         throwStatus(400, "invalid partitionId: " + part)
     const devid = getDeviceIdFromParams(req, part)
-    const dev = await storage.getDevice(devid)
+    const dev = await getDevice(devid)
     if (dev == null) throwStatus(404, "no such device: " + devid.rowKey)
     return dev
 }
@@ -69,10 +82,10 @@ function sanitizeDeviceIdOrThrow(
 }
 
 async function addDevice(id: DeviceId) {
-    let dev = await storage.getDevice(id)
+    let dev = await getDevice(id)
     if (!dev) {
         console.log(`creating device ${fullDeviceId(id)}`)
-        dev = await storage.createDevice(id, {
+        dev = await createDevice(id, {
             key: randomBytes(32).toString("base64"),
         })
     }
@@ -126,11 +139,7 @@ async function patchDevice(id: DeviceId, req: FastifyRequest) {
     if (scriptId != undefined && scriptId != "") {
         checkString(scriptId)
         try {
-            const scr = await storage.getScript(
-                req.partition,
-                scriptId,
-                scriptVersion
-            )
+            const scr = await getScript(req.partition, scriptId, scriptVersion)
             // default to latest version
             if (!scriptVersion) scriptVersion = scr.version
         } catch (e: any) {
@@ -145,9 +154,9 @@ async function patchDevice(id: DeviceId, req: FastifyRequest) {
         }
     }
 
-    await storage.updateDevice(id, d => {
+    await updateDevice(id, d => {
         if (name != undefined) d.name = name
-        if (meta != undefined) d.metaJSON = storage.stringifyMeta(meta)
+        if (meta != undefined) d.metaJSON = stringifyMeta(meta)
         if (scriptId != undefined) {
             d.scriptId = scriptId
             d.scriptVersion = scriptVersion
@@ -204,31 +213,31 @@ export async function initHubRoutes(server: FastifyInstance) {
         if (method) validateMethod(method)
         if (typeof url != "string" || !/^https:\/\//.test(url))
             throwStatus(418, "invalid URL")
-        const key = await storage.addMessageHook(req.partition, {
+        const key = await addMessageHook(req.partition, {
             deviceId,
             method,
             url,
         })
         return reply
             .status(201)
-            .headers({ location: storage.selfUrl() + "/hooks/" + key })
+            .headers({ location: selfUrl() + "/hooks/" + key })
             .send({ id: key })
     })
 
     server.get("/hooks", async req => {
-        return await storage.listMessageHooks(req.partition)
+        return await listMessageHooks(req.partition)
     })
 
     server.delete<{ Params: { hookId: string } }>(
         "/hooks/:hookId",
         async req => {
-            await storage.deleteMessageHook(req.partition, req.params.hookId)
+            await deleteMessageHook(req.partition, req.params.hookId)
             return {}
         }
     )
 
     server.get("/devices", async req => {
-        const infos = await storage.getDeviceList(req.partition)
+        const infos = await getDeviceList(req.partition)
         return infos.map(externalDevice)
     })
 
@@ -239,13 +248,13 @@ export async function initHubRoutes(server: FastifyInstance) {
 
     server.get("/devices/:deviceId/fwd", async req => {
         const devid = getDeviceIdFromParams(req)
-        const dev = await storage.getDevice(devid)
+        const dev = await getDevice(devid)
         return fwdSockConnSettings(dev, "devfwd")
     })
 
     server.get("/devices/:deviceId/logs", async req => {
         const devid = getDeviceIdFromParams(req)
-        const dev = await storage.getDevice(devid)
+        const dev = await getDevice(devid)
         return fwdSockConnSettings(dev, "devlogs")
     })
 
@@ -256,7 +265,7 @@ export async function initHubRoutes(server: FastifyInstance) {
 
     server.delete("/devices/:deviceId", async req => {
         const devid = getDeviceIdFromParams(req)
-        await storage.deleteDevice(devid)
+        await deleteDevice(devid)
         return {}
     })
 }

src/apidevices.ts

@@ -1,6 +1,16 @@
 import { FastifyInstance, FastifyRequest } from "fastify"
-import * as storage from "./storage"
 import { checkString, throwStatus } from "./util"
+import {
+    ScriptBody,
+    ScriptProperties,
+    createScript,
+    deleteScript,
+    getScript,
+    getScriptBody,
+    getScriptVersions,
+    listScripts,
+    updateScript,
+} from "./storage"
 
 function checkScriptId(id: string) {
     if (typeof id != "string" || !/^\d{10}[a-zA-Z]{12}$/.test(id))
@@ -9,15 +19,15 @@ function checkScriptId(id: string) {
 
 export async function initScriptRoutes(server: FastifyInstance) {
     server.get("/scripts", async req => {
-        return await storage.listScripts(req.partition)
+        return await listScripts(req.partition)
     })
 
-    async function getScript(req: FastifyRequest) {
+    async function fetchScript(req: FastifyRequest) {
         const scriptId = (req.params as any).scriptId
         checkScriptId(scriptId)
         const verId = parseInt((req.params as any).version) || undefined
         try {
-            return await storage.getScript(req.partition, scriptId, verId)
+            return await getScript(req.partition, scriptId, verId)
         } catch (e: any) {
             if (e.statusCode == 404)
                 throwStatus(
@@ -28,44 +38,44 @@ export async function initScriptRoutes(server: FastifyInstance) {
         }
     }
 
-    server.get("/scripts/:scriptId", getScript)
+    server.get("/scripts/:scriptId", fetchScript)
     server.get("/scripts/:scriptId/body", async req => {
-        const scr = await getScript(req)
-        return await storage.getScriptBody(scr.id, scr.version)
+        const scr = await fetchScript(req)
+        return await getScriptBody(scr.id, scr.version)
     })
 
     server.put("/scripts/:scriptId/body", async req => {
-        const scr = await getScript(req)
+        const scr = await fetchScript(req)
         const body = verifyBody(req.body)
-        return await storage.updateScript(scr, { body })
+        return await updateScript(scr, { body })
     })
 
     server.get("/scripts/:scriptId/versions", async req => {
-        const scr = await getScript(req)
-        const headers = await storage.getScriptVersions(scr)
+        const scr = await fetchScript(req)
+        const headers = await getScriptVersions(scr)
         return { headers }
     })
 
-    server.get("/scripts/:scriptId/versions/:version", getScript)
+    server.get("/scripts/:scriptId/versions/:version", fetchScript)
     server.get("/scripts/:scriptId/versions/:version/body", async req => {
-        const scr = await getScript(req)
-        return await storage.getScriptBody(scr.id, scr.version)
+        const scr = await fetchScript(req)
+        return await getScriptBody(scr.id, scr.version)
     })
 
-    function verifyBody(body: any): storage.ScriptBody {
+    function verifyBody(body: any): ScriptBody {
         if (typeof body != "object" || Array.isArray(body))
             throwStatus(412, "invalid body type")
         if (JSON.stringify(body).length > 5 * 1024 * 1024)
             throwStatus(413, "body too large")
-        const b = body as storage.ScriptBody
+        const b = body as ScriptBody
         if (!b.program || typeof b.program?.binary?.hex != "string")
             throwStatus(418, "invalid body format")
         return body
     }
 
     function getScriptData(req: FastifyRequest) {
         const reqbody: any = req.body
-        const props: storage.ScriptProperties = {
+        const props: ScriptProperties = {
             name: reqbody.name,
             meta: reqbody.meta,
             body: reqbody.body,
@@ -82,19 +92,19 @@ export async function initScriptRoutes(server: FastifyInstance) {
     }
 
     server.patch("/scripts/:scriptId", async req => {
-        const scr = await getScript(req)
+        const scr = await fetchScript(req)
         const updates = getScriptData(req)
-        return await storage.updateScript(scr, updates)
+        return await updateScript(scr, updates)
     })
 
     server.delete("/scripts/:scriptId", async req => {
-        const scr = await getScript(req)
-        return await storage.deleteScript(req.partition, scr.id)
+        const scr = await fetchScript(req)
+        return await deleteScript(req.partition, scr.id)
     })
 
     server.post("/scripts", async req => {
         const props = getScriptData(req)
         if (!props.body) throwStatus(400, "no body")
-        return await storage.createScript(req.partition, props)
+        return await createScript(req.partition, props)
     })
 }