Fixes #11 #9 #5 #3 and a few other bugs

Author: Qyn

docker-compose.yml

@@ -65,7 +65,7 @@ services:
     networks:
       - internal
     volumes:
-      - /traffic:/traffic:ro
+      - /home/qyn/tulip/traffic:/traffic:ro
     command: "./enricher -eve /traffic/eve.json"
     environment:
       TULIP_MONGO: mongo:27017

frontend/src/api.ts

@@ -14,8 +14,8 @@ export interface Flow {
     // TODO: get this from backend instead of hacky work around
     service_tag: string
     inx: number
-    starred: boolean
-    blocked: boolean
+    parent_id: Id
+    child_id: Id
     tags: string[]
     suricata: number[]
     filename: string
@@ -76,12 +76,10 @@ class TulipApi {
     async _getFlows(query: FlowsQuery, destToService: any) {
 
         // HACK: make starred look like a tag
-        const starred = query.tags.includes("starred");
-        let tags = query.tags.filter(tag => tag !== "starred")
+        let tags = query.tags;
         const hacky_query = {
             ...query,
             tags: tags.length > 0 ? tags : undefined,
-            starred
         }
         // END HACK
 
@@ -110,8 +108,8 @@ class TulipApi {
         const response = await fetch(`${this.API_ENDPOINT}/tags`);
         const tags = await response.json();
 
-        // HACK: make starred look like a tag
-        tags.push("starred")
+        // HACK: make connected flows look like a tag
+        tags.push("connected")
         // END HACK
 
 
@@ -129,8 +127,8 @@ class TulipApi {
         return await response.text()
     }
 
-    async toPythonRequest(body: string, tokenize: boolean) {
-        const response = await fetch(`${this.API_ENDPOINT}/to_python_request?tokenize=${tokenize ? "1" : "0"}`, {
+    async toPythonRequest(body: string, id: string, tokenize: boolean) {
+        const response = await fetch(`${this.API_ENDPOINT}/to_python_request?tokenize=${tokenize ? "1" : "0"}&id=${id}`, {
             method: "POST",
             headers: {
                 "Content-Type": "text/plain;charset=UTF-8"

frontend/src/components/FlowList.tsx

@@ -4,7 +4,7 @@ import {
   useParams,
   useNavigate,
 } from "react-router-dom";
-import { useCallback, useEffect, useState } from "react";
+import { useCallback, useEffect, useState, useRef } from "react";
 import { useAtom, useAtomValue } from "jotai";
 import { Flow, FullFlow, useTulip } from "../api";
 import {
@@ -14,7 +14,7 @@ import {
   END_FILTER_KEY,
 } from "../App";
 
-import { HeartIcon, FilterIcon } from "@heroicons/react/solid";
+import { HeartIcon, FilterIcon, LinkIcon } from "@heroicons/react/solid";
 import {
   HeartIcon as EmptyHeartIcon,
   FilterIcon as EmptyFilterIcon,
@@ -23,7 +23,7 @@ import {
 import classes from "./FlowList.module.css";
 import { format } from "date-fns";
 import useDebounce from "../hooks/useDebounce";
-import { Virtuoso } from "react-virtuoso";
+import { Virtuoso, VirtuosoHandle } from "react-virtuoso";
 import classNames from "classnames";
 import { Tag } from "./Tag";
 import { lastRefreshAtom } from "./Header";
@@ -36,6 +36,11 @@ export function FlowList() {
 
   const [flowList, setFlowList] = useState<Flow[]>([]);
 
+  // Set default flowIndex to Infinity, so that initialTopMostItemIndex != 0 and therefore scrolledToInitialItem != true
+  const [flowIndex, setFlowIndex] = useState<number>(Infinity);
+
+  const virtuoso = useRef<VirtuosoHandle>(null);
+
   const service_name = searchParams.get(SERVICE_FILTER_KEY) ?? "";
   const service = services.find((s) => s.name == service_name);
 
@@ -73,8 +78,13 @@ export function FlowList() {
         service: "", // FIXME
         tags: selectedTags,
       });
+      data.forEach((flow, index)=>  {
+        if(flow._id.$oid === params.id){ setFlowIndex(index)}
+      })
+
       setFlowList(data);
       setLoading(false);
+      
     };
     fetchData().catch(console.error);
   }, [
@@ -84,12 +94,14 @@ export function FlowList() {
     to_filter,
     selectedTags,
     lastRefresh,
+    params,
+    virtuoso
   ]);
 
   const onHeartHandler = useCallback(async (flow: Flow) => {
-    await api.starFlow(flow._id.$oid, !flow.starred);
+    await api.starFlow(flow._id.$oid, !flow.tags.includes("starred"));
     // optimistic update
-    const newFlow = { ...flow, starred: !flow.starred };
+    const newFlow = { ...flow};
     setFlowList((prev) =>
       prev.map((f) => (f._id.$oid === flow._id.$oid ? newFlow : f))
     );
@@ -147,6 +159,8 @@ export function FlowList() {
           "sidebar-loading": loading,
         })}
         data={flowList}
+        ref={virtuoso}
+        initialTopMostItemIndex={flowIndex}
         itemContent={(index, flow) => (
           <Link
             to={`/flow/${flow._id.$oid}?${searchParams}`}
@@ -178,15 +192,14 @@ function FlowListEntry({ flow, isActive, onHeartClick }: FlowListEntryProps) {
 
   const isStarred = flow.tags.includes("starred");
   // Filter tag list for tags that are handled specially
-  const filtered_tag_list = flow.tags.filter((t) => !["starred"].includes(t));
+  const filtered_tag_list = flow.tags.filter((t) => t != "starred");
 
   const duration =
     flow.duration > 10000 ? (
       <div className="text-red-500">&gt;10s</div>
     ) : (
       <div>{flow.duration}ms</div>
     );
-
   return (
     <li
       className={classNames({
@@ -195,17 +208,23 @@ function FlowListEntry({ flow, isActive, onHeartClick }: FlowListEntryProps) {
     >
       <div className="flex">
         <div
-          className="w-5 ml-2 mr-4 self-center shrink-0"
+          className="w-5 ml-1 mr-1 self-center shrink-0"
           onClick={() => {
             onHeartClick(flow);
           }}
         >
-          {flow.starred ? (
+          {flow.tags.includes("starred") ? (
             <HeartIcon className="text-red-500" />
           ) : (
             <EmptyHeartIcon />
           )}
         </div>
+
+        <div className="w-5 mr-2 self-center shrink-0">
+          {flow.child_id.$oid != "000000000000000000000000" || flow.parent_id.$oid != "000000000000000000000000" ? (
+            <LinkIcon className="text-blue-500" />
+          ) : undefined}
+        </div>
         <div className="flex-1 shrink">
           <div className="flex">
             <div className="shrink-0">

frontend/src/pages/FlowView.tsx

@@ -1,4 +1,4 @@
-import { useParams } from "react-router-dom";
+import { useSearchParams, Link, useParams } from "react-router-dom";
 import React, { useEffect, useState } from "react";
 import { useTulip, FlowData, FullFlow } from "../api";
 import { Buffer } from "buffer";
@@ -84,14 +84,14 @@ function WebFlow({ flow }: { flow: FlowData }) {
   );
 }
 
-function PythonRequestFlow({ flow }: { flow: FlowData }) {
+function PythonRequestFlow({ full_flow, flow }: {full_flow:FullFlow, flow: FlowData }) {
   const [data, setData] = useState("");
 
   const { api } = useTulip();
 
   useEffect(() => {
     const fetchData = async () => {
-      const data = await api.toPythonRequest(btoa(flow.data), true);
+      const data = await api.toPythonRequest(btoa(flow.data), full_flow._id.$oid,true);
       setData(data);
     };
     // TODO proper error handling
@@ -102,6 +102,7 @@ function PythonRequestFlow({ flow }: { flow: FlowData }) {
 }
 
 interface FlowProps {
+  full_flow: FullFlow;
   flow: FlowData;
   delta_time: number;
 }
@@ -140,7 +141,7 @@ function detectType(flow: FlowData) {
   return "Plain";
 }
 
-function Flow({ flow, delta_time }: FlowProps) {
+function Flow({ full_flow, flow, delta_time }: FlowProps) {
   const formatted_time = format(new Date(flow.time), "HH:mm:ss:SSS");
   const displayOptions = ["Plain", "Hex", "Web", "PythonRequest"];
 
@@ -183,7 +184,7 @@ function Flow({ flow, delta_time }: FlowProps) {
         {displayOption === "Plain" && <TextFlow flow={flow}></TextFlow>}
         {displayOption === "Web" && <WebFlow flow={flow}></WebFlow>}
         {displayOption === "PythonRequest" && (
-          <PythonRequestFlow flow={flow}></PythonRequestFlow>
+          <PythonRequestFlow flow={flow} full_flow={full_flow}></PythonRequestFlow>
         )}
       </div>
     </div>
@@ -263,6 +264,7 @@ function FlowOverview({ flow }: { flow: FullFlow }) {
 function Header() {}
 
 export function FlowView() {
+  let [searchParams] = useSearchParams();
   const params = useParams();
   const [flow, setFlow] = useState<FullFlow>();
 
@@ -318,17 +320,30 @@ export function FlowView() {
           </button>
         </div>
       </div>
+      {flow?.parent_id?.$oid != "000000000000000000000000"? <Link
+            to={`/flow/${flow.parent_id.$oid}?${searchParams}`}
+            key={flow.parent_id.$oid}
+            className="focus-visible:rounded-md"
+          >Parent</Link>: undefined}
+
       {flow ? <FlowOverview flow={flow}></FlowOverview> : undefined}
       {flow?.flow.map((flow_data, i, a) => {
         const delta_time = a[i].time - (a[i - 1]?.time ?? a[i].time);
         return (
           <Flow
             flow={flow_data}
             delta_time={delta_time}
+            full_flow={flow}
             key={flow._id.$oid + " " + i}
           ></Flow>
         );
       })}
+
+      {flow?.child_id?.$oid != "000000000000000000000000" ? <Link
+            to={`/flow/${flow.child_id.$oid}?${searchParams}`}
+            key={flow.child_id.$oid}
+            className="focus-visible:rounded-md"
+          >Child</Link>: undefined}
     </div>
   );
 }

services/README.md

@@ -18,7 +18,6 @@ Each document will have:
         "dst_ip": "127.0.0.1",
         "dst_port": 1234,
         "contains_flag": //true if the importer have found that the flow contains a flag based on the env var regex
-        "starred": //if the flow is starred
         "flow": [
             {
                 "data": "...", //printable data
@@ -59,9 +58,6 @@ Returns the all document with `flow_id` id, including the field `flow`
 ##### GET /star/(flow_id)/(0,1)
 Set the flow favourite (1) or not (0)
 
-##### POST /starred
-Returns a list of document like `/query` endpoint, but only with starred items.
-
 ##### POST /to_python_request/(tokenize)
 convert the request to python syntax. Tokenize is used to toggle the auto-parsing of args.
 

services/data2req.py

@@ -48,7 +48,7 @@ def send_error(self, code, message):
         self.error_message = message
 
 # tokenize used for automatically fill data param of request
-def convert_http_requests(raw_request, tokenize=True, use_requests_session=False):
+def convert_http_requests(raw_request, flow, tokenize=True, use_requests_session=False):
     request = HTTPRequest(raw_request)
 
     data = {}
@@ -96,8 +96,9 @@ def convert_http_requests(raw_request, tokenize=True, use_requests_session=False
 
     rtemplate = Environment(loader=BaseLoader()).from_string("""import os
 import requests
+import sys
 
-host = os.getenv("TARGET_IP")
+host = sys.argv[1]
 {% if use_requests_session %}
 s = requests.Session()
 
@@ -107,12 +108,13 @@ def convert_http_requests(raw_request, tokenize=True, use_requests_session=False
 {% endif %}
 data = {{data}}
 
-{% if use_requests_session %}s{% else %}requests{% endif %}.{{request.command.lower()}}("http://{}{{request.path}}".format(host), {{data_param_name}}=data{% if not use_requests_session %}, headers=headers{% endif %})""")
+{% if use_requests_session %}s{% else %}requests{% endif %}.{{request.command.lower()}}("http://{}:{{port}}{{request.path}}".format(host), {{data_param_name}}=data{% if not use_requests_session %}, headers=headers{% endif %})""")
 
     return rtemplate.render(
             headers=str(dict(headers)),
             data=data,
             request=request,
             data_param_name=data_param_name,
             use_requests_session=use_requests_session,
+            port=flow["dst_port"]
         )

services/db.py

@@ -59,10 +59,6 @@ def getFlowList(self, filters):
         if "from_time" in filters and "to_time" in filters:
             f["time"] = {"$gte": int(filters["from_time"]),
                          "$lt": int(filters["to_time"])}
-        if "starred" in filters:
-            f["starred"] =  bool(filters["starred"])
-        if "blocked" in filters:
-            f["blocked"] =  bool(filters["blocked"])
         if "tags" in filters:
             f["tags"] = {
                 "$all": [str(elem) for elem in filters["tags"]]
@@ -92,7 +88,10 @@ def getFlowDetail(self, id):
         return ret
 
     def setStar(self, flow_id, star):
-        self.pcap_coll.find_one_and_update({"_id": ObjectId(flow_id)}, {"$set": {"starred":  star}})
+        if star:
+            self.pcap_coll.find_one_and_update({"_id": ObjectId(flow_id)}, {"$push": {"tags": "starred"}})
+        else:
+            self.pcap_coll.find_one_and_update({"_id": ObjectId(flow_id)}, {"$pull": {"tags": "starred"}})
 
     def isFileAlreadyImported(self, file_name):
         return self.file_coll.find({"file_name": file_name}).count() != 0

services/flow2pwn.py

@@ -40,8 +40,10 @@ def flow2pwn(flow):
     port = flow["dst_port"]
 
     script = """from pwn import *
+import sys
 
-proc = remote('{}', {})
+host = sys.argv[1]
+proc = remote(host, {})
 """.format(ip, port)
 
     for message in flow['flow']: