feat(fabric): add fabric support to run app or other tasks

- add fabfile.py
- set message when no result found in website
- set LOGS_DIR environment variable

Author: kodekracker

README.md

@@ -5,6 +5,7 @@ A web crawler/scraper to find the broken links in the targeted seed url based on
 
 ##Installation
 1. Redis
+3. Fabric
 2. Python 2.7+
 
 ##Instructions
@@ -26,18 +27,26 @@ A web crawler/scraper to find the broken links in the targeted seed url based on
     export SMTP_PASSWORD='smtp-password'
 ```
 
+4. Also, set the one more environmnet variable to save **`Logs`** of the app in defined location.
+```python
+    # your shell config file
+    export LOGS_DIR='path/to/logs'
+```
+
 ##Commands
+Note:- First install *`Fabric`* to run below commands
+
 To run a gui app :
 ```
-    $ python rottoscraper/run.py app
+    $ fab app
 ```
 To run a dispatcher :
 ```
-    $ python rottoscraper/run.py dispatcher
+    $ fab dispatcher
 ```
 To run a worker :
 ```
-    $ python rotttoscraper/worker.py
+    $ fab worker
 ```
 ##Developer
 1. [Akshay Pratap Singh](https://www.facebook.com/AKSHAYPRATAP007)

fabfile.py

@@ -0,0 +1,13 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from fabric.api import local
+
+def app():
+    local('python rottoscraper/run.py app')
+
+def dispatcher():
+    local('python rottoscraper/run.py dispatcher')
+
+def worker():
+    local('python rottoscraper/worker.py')

rottoscraper/config.py

@@ -28,3 +28,6 @@
 # SMTP Cerendentials
 SMTP_USER = os.getenv('SMTP_USER', None)
 SMTP_PASSWORD = os.getenv('SMTP_PASSWORD', None)
+
+# Logs DIR Path
+LOGS_DIR = os.getenv('LOGS_DIR', 'logs/')

rottoscraper/gui/static/html/result.html

@@ -36,10 +36,10 @@
             </table>
         </div>
         <div class="result-content fancy-box">
-            <div class="msg" ng-show="website.result.length==0">
+            <div class="msg" ng-if="!website.result">
                 <p>No Rotto Links Page Found</p>
             </div>
-            <div class="result-content-row"  ng-repeat="rottopage in website.result">
+            <div class="result-content-row" ng-if='website.result' ng-repeat="rottopage in website.result">
 
                 <table class="table">
                     <tr>

rottoscraper/logger.py

@@ -9,10 +9,13 @@
 from logbook import FileHandler
 from logbook import Logger
 
+from config import LOGS_DIR
+
 log = Logger('scraper')
 
 # Create a logs direcory if not exist
-if not os.path.exists('logs'):
-    os.makedirs('logs')
-file_handler = FileHandler('logs/app.log', level=logbook.DEBUG)
+if not os.path.exists(LOGS_DIR):
+    os.makedirs(LOGS_DIR)
+log_file_name = 'rottoscraper.log'
+file_handler = FileHandler(LOGS_DIR + log_file_name, level=logbook.DEBUG)
 file_handler.push_application()

rottoscraper/scraper/aho.py

@@ -89,7 +89,7 @@ def search_keywords(self, text=None):
             while trans is None:
                 # trans=currentNode.GetTransition(text[index])
                 for x in currentNode.transitions:
-                    if unicode(x.char) == c:
+                    if x.char == c:
                         trans = x
                 if currentNode == self.root:
                     break

rottoscraper/scraper/utils.py

@@ -87,7 +87,7 @@ def get_plain_text(html):
         Return the plain text in utf-8 encoding from a html
     """
     raw_text = nltk.clean_html(html)
-    text = u' '.join(raw_text.split()).encode('utf-8').lower()
+    text = u' '.join(raw_text.split()).lower()
     return text
 
 def get_all_links(html):