Main

.gitignore

@@ -2,3 +2,9 @@ __pycache__/
 model*/*
 env/*
 .DS_Store
+Images/tmp.png
+Images/screenshot.png
+test.py
+Images/1680x1050_grid.png
+Images/screenshot.png
+Images/tmp.png

docs/userguide.md

@@ -7,6 +7,7 @@ Sidekick takes voice commands and converts them to actions on the computer. It u
 - `text` - this mode transcribes spoken speech to text
 - `alpha` - this mode provides the ability to write individual letters, numbers, and punctuation
 - `pause` - in this mode no commands are processed (convenient if afk). 
+- `volume` - this mode allows the user to control the mouse with the volume of their voice
 
 In each mode certain keywords are linked to certain actions. To switch between modes, simply say `command`, `mouse`, `text`, or `alpha`. Say `pause` once to pause. Say `time to work` to restart back in the `command` state. 
 
@@ -56,6 +57,11 @@ Some commands are stateless, in that they function no matter what state/mode you
 - `hold` - holds down left mouse button until you say another word - useful for drag and drop or on Mac when need to hold and release to switch windows
 - `hot` - hot key press (ex: `hot control alt delete go` presses `ctrl alt delete`) - using the word `apple` for the command key (ex: `hot apple f go` presses `command f`)
 
+#### New Stateful commands
+- `screenshot` opens a window that shows a real time screenshot with a red grid overlay. This overlay corresponds to the grid-based mouse control. To turn of screenshot, say the `screenshot` keyword again
+- `overlay` overlays a red grid over the entire screen. To turn off the overlay, close the window manually
+
+
 #### Examples
 
 - `scroll up 1 2 1` - will scroll up 1, then 2, then 1 again - number can be repeated without repeating entire command
@@ -115,4 +121,16 @@ The alpha mode enables punctuation as well as single alphanumeric characters.
 
 #### Examples
 
-- `cap hello alpha comma text how are you alpha question` - will produce the text 'Hello, how are you?'
+- `cap hello alpha comma text how are you alpha question` - will produce the text 'Hello, how are you?'
+
+## Volume
+Volume mode allows the mouse to be controlled using the volume of your voice
+- `up` switch to vertical movement (default)
+- `left` switch to horizontal movement
+- `slow` mouse moves at a slow speed
+- `medium` mouse moves at a medium speed (default)
+- `fast` mouse moves at a fast speed
+- `stop` exits to command mode
+
+
+

overlay.py

@@ -0,0 +1,51 @@
+import sys
+import time
+import threading
+from PyQt5.QtWidgets import QApplication, QWidget, QLabel
+from PyQt5.QtGui import QIcon, QPixmap
+from PyQt5 import QtCore
+
+# References: https://pythonspot.com/pyqt5-image/
+#             https://stackoverflow.com/questions/1925015/pyqt-always-on-top
+#             https://stackoverflow.com/questions/37941039/pyqt-transparent-background-image-partially-black
+
+class Grid_Overlay(QWidget):
+
+    def __init__(self):
+        super().__init__()
+        self.title = 'Screenshot'
+        self.left = 10
+        self.top = 10
+        self.width = 640
+        self.height = 480
+        self.grid = "Images/1920x1080_grid.png"
+        self.initUI()
+
+    def initUI(self):
+        self.setWindowTitle(self.title)
+        self.setGeometry(self.left, self.top, self.width, self.height)
+        self.setWindowFlag(QtCore.Qt.FramelessWindowHint)
+        self.setWindowFlag(QtCore.Qt.WindowStaysOnTopHint)
+        self.setAttribute(QtCore.Qt.WA_TranslucentBackground, True)
+       # self.setAttribute(QtCore.Qt.WindowStaysOnTopHint, True)
+
+        # Create widget
+        label = QLabel(self)
+        pixmap = QPixmap(self.grid)
+        label.setPixmap(pixmap)
+        self.resize(pixmap.width(),pixmap.height())
+        self.show()
+    def set_grid(self, filename):
+        self.grid = filename
+
+def overlay(filename):
+    app = QApplication([])
+    ex = Grid_Overlay()
+    ex.set_grid(filename)
+    app.exec_()
+    app.quit()
+
+def main():
+    overlay()
+if __name__ == '__main__':
+    main()

parsepackage/command_parser.py

@@ -16,15 +16,20 @@
 along with this program.  If not, see <http://www.gnu.org/licenses/>.
 '''
 from actions import *
+from screenshot import *
+from overlay import overlay
 import string
-
+import threading
+from os.path import exists
 
 class CommandParser:
     def __init__(self, system, steps):
         self.os = system
         self.steps = steps 
         self.tempvar = ""
-
+        self.stop_screenshot = [False]
+        self.screenshot_started = False
+        self.screen_size = (1920, 1080)
         self.keys = ['a', 'b', 'c', 'd', 'e','f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o',
 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z','alt','delete','control','shift','tab','apple']
 
@@ -125,6 +130,8 @@ def __init__(self, system, steps):
             "west",
             "save",
             "scroll",
+            "screenshot",
+            "overlay",
         ]
 
         self.commandlist = (
@@ -381,6 +388,35 @@ def evaluate_command(self, command_buffer):
             else:
                 hotKeyPress(["ctrl", "s"])
             command_buffer = []
+        elif command_buffer[0] == "line": 
+            hotKeyPress(["end"])
+            hotKeyPress(["shift", "home"])
+            command_buffer = []
+        elif command_buffer[0] == "copy line": 
+            hotKeyPress(["end"])
+            hotKeyPress(["shift", "home"])
+            if self.os == "Darwin":
+                hotKeyPress(["command", "c"])
+            else:
+                hotKeyPress(["ctrl", "c"])
+            command_buffer = []
+        elif command_buffer[0] == "cut line": 
+            hotKeyPress(["end"])
+            hotKeyPress(["shift", "home"])
+            if self.os == "Darwin":
+                hotKeyPress(["command", "x"])
+            else:
+                hotKeyPress(["ctrl", "x"])
+            command_buffer = []
+        elif command_buffer[0] == "loop": 
+            pyautogui.write("for (int i = 0; i < N; i++) {")
+            hotKeyPress(["enter"])
+            pyautogui.write("continue;")
+            hotKeyPress(["enter"])
+            pyautogui.write("}")
+            hotKeyPress(["enter"])
+            hotKeyPress(["up", "up", "end"])
+            command_buffer = []
         elif command_buffer[0] == "switch":
 
             if self.os == "Darwin":
@@ -548,6 +584,36 @@ def evaluate_command(self, command_buffer):
                     return self.handle_invalid_command(
                         command_buffer[1], command_buffer
                     )
+        elif command_buffer[0] == "screenshot":
+            if self.screenshot_started == False and self.stop_screenshot[0] == True:
+                self.stop_screenshot[0] = False
+
+            if self.screenshot_started == False:
+                print(command_buffer)
+                w = self.screen_size[0]
+                h = self.screen_size[1]
+                grid = "Images/{}x{}_grid.png".format(w,h)
+                if not exists(grid):
+                    create_gridlines(w, h)
+
+                p = threading.Thread(target=take_screenshot, args=(w, h, grid, self.stop_screenshot))
+                p.start()
+                self.screenshot_started = True
+            else:
+                self.stop_screenshot[0] = True
+                self.screenshot_started = False
+            command_buffer=[]
+
+        elif command_buffer[0] == "overlay":
+            print("Showing grid overlay. Close the window manually to continue using Sidekick.")
+            w = self.screen_size[0]
+            h = self.screen_size[1]
+            grid = "Images/{}x{}_grid.png".format(w,h)
+            if not exists(grid):
+                create_gridlines(w, h)           
+            overlay(grid)
+            command_buffer=[]
+
         else:
             command_buffer = []
 

parsepackage/horizontal_parser.py

@@ -0,0 +1,112 @@
+'''
+Sidekick
+Copyright (C) 2021 UT-Battelle - Created by Sean Oesch
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+'''
+from actions import *
+import threading
+import math
+import audioop
+
+
+
+class HorizontalParser:
+    def __init__(self, system, steps):
+        self.volumeStarted = False
+        self.os = system
+        self.steps = steps
+        self.stopVolume = True
+
+        self.commands = [
+            "stop",
+            "snail",
+            "slow",
+            "fast",
+            "medium",
+            "up",
+            "down",
+            "counter",
+            "clock",
+            "north",
+            "south",
+            "east",
+            "west",
+            "one",
+            "two",
+            "three",
+            "four",
+            "northeast",
+            "northwest",
+            "southeast",
+            "southwest",
+        ]
+    def set_threshold(self, threshold):
+        self.threshold = threshold
+
+        self.midpoint = (self.threshold + 55) /2
+
+    def set_audio_stream(self, stream):
+        self.stream = stream
+
+    def evaluate_volume(self, command_buffer):
+        if not self.volumeStarted:
+
+            self.stopVolume = False
+            self.magnitude = 5  # in pixels
+            self.sleep = 0.2
+            self.setVolumeCoord(90)
+
+        data = self.stream.read(4000,exception_on_overflow = False)
+        # calculate decibels
+        dB = 20 * math.log10(audioop.rms(data,2)+1)
+
+       # if len(command_buffer) > 0:     
+        """print("Volume " + str(dB))
+        if dB < 45:
+            print("MOM")
+            self.setVolumeCoord(self.currentangle + 15)
+        elif dB >= 45:
+            print("WOW")
+            self.setVolumeCoord(self.currentangle - 15)"""
+        command_buffer = []
+
+        if not self.volumeStarted:
+            self.startVolume()
+
+        return [command_buffer, "volume"]
+
+    def startVolume(self):
+        thread = threading.Thread(target=self.volume_thread)
+        thread.daemon = True
+        thread.start()
+        self.volumeStarted = True
+
+    def setVolumeCoord(self, degrees):
+        print("start")
+        self.currentangle = degrees
+        self.x = self.magnitude * math.cos(math.radians(degrees))
+        print(self.x)
+        self.y = -1 * self.magnitude * math.sin(math.radians(degrees))
+        print(self.y)
+        return
+
+    def volume_thread(self):
+        while True:
+            if self.stopVolume:
+                self.volumeStarted = False
+                break
+            else:
+                moveMouse(self.x, self.y)
+                time.sleep(self.sleep)