REVERTED - added similarity filter/comparision

frontend/src/api.ts

@@ -36,6 +36,8 @@ export const tulipApi = createApi({
           ...query,
           includeTags: query.includeTags.length > 0 ? query.includeTags : undefined,
           excludeTags: query.excludeTags.length > 0 ? query.excludeTags : undefined,
+          includeFuzzyHashes: query.includeFuzzyHashes.length > 0 ? query.includeFuzzyHashes : undefined,
+          excludeFuzzyHashes: query.excludeFuzzyHashes.length > 0 ? query.excludeFuzzyHashes : undefined,
         }),
       }),
     }),

frontend/src/components/Corrie.tsx

@@ -7,7 +7,7 @@ import {
   START_FILTER_KEY,
   END_FILTER_KEY,
   CORRELATION_MODE_KEY,
-  FLOW_LIST_REFETCH_INTERVAL_MS,
+  FLOW_LIST_REFETCH_INTERVAL_MS, SIMILARITY_FILTER_KEY,
 } from "../const";
 import useDebounce from "../hooks/useDebounce";
 
@@ -29,15 +29,16 @@ export const Corrie = () => {
   const includeTags = useAppSelector((state) => state.filter.includeTags);
   const excludeTags = useAppSelector((state) => state.filter.excludeTags);
   const filterTags = useAppSelector((state) => state.filter.filterTags);
-  const filterFlags = useAppSelector((state) => state.filter.filterFlags);
-  const filterFlagids = useAppSelector((state) => state.filter.filterFlagids);
+  const includeFuzzyHashes = useAppSelector((state) => state.filter.includeFuzzyHashes);
+  const excludeFuzzyHashes = useAppSelector((state) => state.filter.excludeFuzzyHashes);
 
   const [searchParams, setSearchParams] = useSearchParams();
 
   const service_name = searchParams.get(SERVICE_FILTER_KEY) ?? "";
   const service = services && services.find((s) => s.name == service_name);
 
   const text_filter = searchParams.get(TEXT_FILTER_KEY) ?? undefined;
+  const similarity = searchParams.get(SIMILARITY_FILTER_KEY) ?? undefined;
   const from_filter = searchParams.get(START_FILTER_KEY) ?? undefined;
   const to_filter = searchParams.get(END_FILTER_KEY) ?? undefined;
 
@@ -54,8 +55,9 @@ export const Corrie = () => {
       includeTags: includeTags,
       excludeTags: excludeTags,
       tags: filterTags,
-      flags: filterFlags,
-      flagids: filterFlagids,
+      similarity: similarity,
+      includeFuzzyHashes: includeFuzzyHashes,
+      excludeFuzzyHashes: excludeFuzzyHashes
     },
     {
       refetchOnMountOrArgChange: true,

frontend/src/components/FlowList.tsx

@@ -12,10 +12,10 @@ import {
   TEXT_FILTER_KEY,
   START_FILTER_KEY,
   END_FILTER_KEY,
-  FLOW_LIST_REFETCH_INTERVAL_MS,
+  FLOW_LIST_REFETCH_INTERVAL_MS, SIMILARITY_FILTER_KEY,
 } from "../const";
 import { useAppSelector, useAppDispatch } from "../store";
-import { toggleFilterTag } from "../store/filter";
+import {toggleFilterFuzzyHashes, toggleFilterTag} from "../store/filter";
 
 import { HeartIcon, FilterIcon, LinkIcon } from "@heroicons/react/solid";
 import { HeartIcon as EmptyHeartIcon } from "@heroicons/react/outline";
@@ -44,10 +44,12 @@ export function FlowList() {
   const { data: services } = useGetServicesQuery();
 
   const filterTags = useAppSelector((state) => state.filter.filterTags);
-  const filterFlags = useAppSelector((state) => state.filter.filterFlags);
-  const filterFlagids = useAppSelector((state) => state.filter.filterFlagids);
   const includeTags = useAppSelector((state) => state.filter.includeTags);
   const excludeTags = useAppSelector((state) => state.filter.excludeTags);
+  const includeFuzzyHashes = useAppSelector((state) => state.filter.includeFuzzyHashes);
+  const excludeFuzzyHashes = useAppSelector((state) => state.filter.excludeFuzzyHashes);
+  const fuzzyHashes = useAppSelector((state) => state.filter.fuzzyHashes);
+    const fuzzyHashIds = useAppSelector((state) => state.filter.fuzzyHashIds);
 
   const dispatch = useAppDispatch();
 
@@ -61,6 +63,7 @@ export function FlowList() {
   const service = services?.find((s) => s.name == service_name);
 
   const text_filter = searchParams.get(TEXT_FILTER_KEY) ?? undefined;
+  const similarity = searchParams.get(SIMILARITY_FILTER_KEY) ?? undefined;
   const from_filter = searchParams.get(START_FILTER_KEY) ?? undefined;
   const to_filter = searchParams.get(END_FILTER_KEY) ?? undefined;
 
@@ -75,10 +78,11 @@ export function FlowList() {
       to_time: to_filter,
       service: "", // FIXME
       tags: filterTags,
-      flags: filterFlags,
-      flagids: filterFlagids,
       includeTags: includeTags,
-      excludeTags: excludeTags
+      excludeTags: excludeTags,
+      similarity: similarity,
+      includeFuzzyHashes: includeFuzzyHashes,
+      excludeFuzzyHashes: excludeFuzzyHashes
     },
     {
       refetchOnMountOrArgChange: true,
@@ -197,6 +201,24 @@ export function FlowList() {
             </div>
           </div>
         )}
+        {showFilters && (
+            <div className="border-t-gray-300 border-t p-2">
+              <p className="text-sm font-bold text-gray-600 pb-2">
+                Similarity filter
+              </p>
+              <div className="flex gap-2 flex-wrap">
+                {(fuzzyHashes ?? []).map((fuzzyHash, i) => (
+                    <Tag
+                        key={fuzzyHashIds[i]}
+                        tag={fuzzyHashIds[i]}
+                        disabled={!includeFuzzyHashes.includes(fuzzyHash)}
+                        excluded={excludeFuzzyHashes.includes(fuzzyHash)}
+                        onClick={() => dispatch(toggleFilterFuzzyHashes([fuzzyHash, fuzzyHashIds[i]]))}
+                    ></Tag>
+                ))}
+              </div>
+            </div>
+        )}
       </div>
       <div></div>
       <Virtuoso
@@ -213,9 +235,12 @@ export function FlowList() {
             to={`/flow/${flow._id.$oid}?${searchParams}`}
             onClick={() => setFlowIndex(index)}
             key={flow._id.$oid}
-            className="focus-visible:rounded-md"
-            //style={{ paddingTop: '1em' }}
-          >
+          className={classNames({
+            "flex-1": true,
+            [classes.list_container]: true,
+            "sidebar-loading": isLoading,
+          })}
+            >
             <FlowListEntry
               key={flow._id.$oid}
               flow={flow}
@@ -249,11 +274,24 @@ function FlowListEntry({ flow, isActive, onHeartClick }: FlowListEntryProps) {
     ) : (
       <div>{flow.duration}ms</div>
     );
-  return (
+
+
+  const DEFAULT = [243, 244, 246]
+  const GREEN = [134, 239, 172]
+  const RED = [252, 165, 165]
+
+  var color: number[]
+  if (flow.similarity != undefined)
+    color = GREEN.map((g, i) => ((g*flow.similarity) + (RED[i]*(1-flow.similarity))));
+  else
+    color = DEFAULT;
+
+    return (
     <li
       className={classNames({
         [classes.active]: isActive,
       })}
+      style={{backgroundColor: `rgb(${color[0]} ${color[1]} ${color[2]} / var(--tw-bg-opacity))`}}
     >
       <div className="flex">
         <div

frontend/src/components/Header.tsx

@@ -10,14 +10,14 @@ import {
 import ReactDiffViewer from "react-diff-viewer";
 
 import {
-  END_FILTER_KEY,
-  SERVICE_FILTER_KEY,
-  START_FILTER_KEY,
-  TEXT_FILTER_KEY,
-  FIRST_DIFF_KEY,
-  SECOND_DIFF_KEY,
-  SERVICE_REFETCH_INTERVAL_MS,
-  TICK_REFETCH_INTERVAL_MS,
+    END_FILTER_KEY,
+    SERVICE_FILTER_KEY,
+    START_FILTER_KEY,
+    TEXT_FILTER_KEY,
+    FIRST_DIFF_KEY,
+    SECOND_DIFF_KEY,
+    SERVICE_REFETCH_INTERVAL_MS,
+    TICK_REFETCH_INTERVAL_MS, SIMILARITY_FILTER_KEY,
 } from "../const";
 import {
   useGetFlowQuery,
@@ -318,6 +318,45 @@ function Diff() {
   );
 }
 
+function SimilaritySlider() {
+    let [searchParams, setSearchParams] = useSearchParams();
+    // Initialize state to keep track of the value
+    const [value, setValue] = useState(searchParams.get(SIMILARITY_FILTER_KEY) ?? 90); // You can set the initial value to whatever you prefer
+
+    // Function to update the state based on input changes
+    const handleChange = (event:any) => {
+        setValue(event.target.value);
+        searchParams.set(SIMILARITY_FILTER_KEY, event.target.value)
+        setSearchParams(searchParams);
+    };
+
+    return (
+        <div className="flex items-center">
+            <div className="pr-3">
+                <span>
+                    Similarity:
+                </span>
+                <input
+                    style={{paddingTop:0, paddingBottom:0}}
+                    type="range"
+                    min="0" // Set the minimum value of the range
+                    max="100" // Set the maximum value of the range
+                    value={value} // Bind the range input to the value in the state
+                    onChange={handleChange} // Update the state when the range value changes
+                />
+            </div>
+            <input
+                className="w-20"
+                type="number"
+                min="0" // Set the minimum value for the number input
+                max="100" // Set the maximum value for the number input
+                value={value} // Bind the number input to the value in the state
+                onChange={handleChange} // Update the state when the number value changes
+            />
+        </div>
+    );
+}
+
 export function Header() {
   let [searchParams] = useSearchParams();
   const { setToLastnTicks, currentTick, setTimeParam } = useMessyTimeStuff();
@@ -330,17 +369,17 @@ export function Header() {
     setTimeParam("", END_FILTER_KEY);
   });
 
-  return (
-    <>
-      <Link to={`/?${searchParams}`}>
-        <div className="header-icon">🌷</div>
-      </Link>
-      <div>
-        <TextSearch></TextSearch>
-      </div>
-      <div>
-        <Suspense>
-          <ServiceSelection></ServiceSelection>
+    return (
+        <>
+            <Link to={`/?${searchParams}`}>
+                <div className="header-icon">🌷</div>
+            </Link>
+            <div>
+                <TextSearch></TextSearch>
+            </div>
+            <div>
+                <Suspense>
+                    <ServiceSelection></ServiceSelection>
         </Suspense>
       </div>
       <div>
@@ -362,6 +401,7 @@ export function Header() {
           Graph view
         </div>
       </Link>
+        <SimilaritySlider></SimilaritySlider>
       <div className="ml-auto mr-4" style={{ display: "flex" }}>
         <div className="mr-4">
           <FirstDiff />

frontend/src/const.ts

@@ -2,6 +2,7 @@ export const API_BASE_PATH = "/api";
 
 export const TEXT_FILTER_KEY = "text";
 export const SERVICE_FILTER_KEY = "service";
+export const SIMILARITY_FILTER_KEY = "similarity";
 export const START_FILTER_KEY = "start";
 export const END_FILTER_KEY = "end";
 export const FIRST_DIFF_KEY = "first";