diff --git a/.env.example b/.env.example index 2d5403b..cb488a6 100644 --- a/.env.example +++ b/.env.example @@ -2,44 +2,96 @@ # Tulip config ############################## -# The connection string to connect to the mongo bd -TULIP_MONGO="mongo:27017" +# Timescale connection +TIMESCALE="postgres://tulip@timescale:5432/tulip" + # The location of your pcaps as seen by the host TRAFFIC_DIR_HOST="./services/test_pcap" + # The location of your pcaps (and eve.json), as seen by the container TRAFFIC_DIR_DOCKER="/traffic" +# Visualizer +VISUALIZER_URL="http://scraper.example.com" + ############################## # Game config ############################## # Start time of the CTF (or network open if you prefer) -TICK_START="2018-06-27T13:00+02:00" +TICK_START="2024-11-30T13:00:00Z" + # Tick length in ms TICK_LENGTH=180000 + # The flag format in regex FLAG_REGEX="[A-Z0-9]{31}=" +# VM IP (inside gamenet) +# Currently ignored unless FLAGID_SCRAPE is set +VM_IP="10.10.3.1" +TEAM_ID="3" + ############################## # PCAP_OVER_IP CONFIGS ############################## +# Enable pcap-over-ip and choose server endpoint +# Empty value = disabled +PCAP_OVER_IP= #PCAP_OVER_IP="host.docker.internal:1337" -# # For multiple PCAP_OVER_IP you can comma separate +# For multiple PCAP_OVER_IP you can comma separate #PCAP_OVER_IP="host.docker.internal:1337,otherhost.com:5050" +############################## +# DUMP_PCAPS CONFIGS +############################## + +# Enable pcap dumping and select target location +# Empty value = disabled +DUMP_PCAPS= +#DUMP_PCAPS="/traffic" + +# Dumping options +# Ignored unless DUMP_PCAPS is set +DUMP_PCAPS_INTERVAL="1m" +DUMP_PCAPS_FILENAME="2006-01-02_15-04-05.pcap" + ############################## # FLAGID CONFIGS ############################## -# # enable flagid scrapping -# FLAGID_SCRAPE=1 -# # enable flagid scanning -# FLAGID_SCAN=1 -# # Flag Lifetime in Ticks (-1 for no check, pls don't use outside testing) -# FLAG_LIFETIME=-1 -# # Flagid endpoint currently Testendpoint in docker compose -# FLAGID_ENDPOINT="http://flagidendpoint:8000/flagids.json" -# # VM IP (inside gamenet) -# VM_IP="10.10.3.1" -# TEAM_ID="10.10.3.1" +# Enable flagid scrapping +# Empty value = disabled +FLAGID_SCRAPE= +#FLAGID_SCRAPE=1 + +# Enable flagid scanning - Tags flag ids in traffic +# Empty value = disabled +# Does nothing unless FLAGID_SCRAPE is set +FLAGID_SCAN= +#FLAGID_SCAN=1 + +# Flag lifetime in ticks +# Empty value = Fallback to TICK_LENGTH +# -1 = No check, pls don't use outside testing +FLAG_LIFETIME= +#FLAG_LIFETIME=-1 +#FLAG_LIFETIME=5 + +# Flagid endpoint +# Default value is a test container in docker-compose-test.yml, change this for production +# Ignored unless FLAGID_SCRAPE is set +FLAGID_ENDPOINT="http://flagidendpoint:8000/flagids.json" + +############################## +# FLAG_VALIDATOR CONFIGS +############################## + +# Enables flag validation / fake flag feature. Must be one of: faust, enowars, eno, itad +# Empty value = disabled +FLAG_VALIDATOR_TYPE= + +# Some flag validators can make use of (our) team number/ID +# Ignored unless FLAG_VALIDATOR_TYPE is set +FLAG_VALIDATOR_TEAM=42 diff --git a/.gitignore b/.gitignore old mode 100755 new mode 100644 diff --git a/README.md b/README.md old mode 100755 new mode 100644 index a892ded..d789f89 --- a/README.md +++ b/README.md @@ -2,6 +2,23 @@ Tulip is a flow analyzer meant for use during Attack / Defence CTF competitions. It allows players to easily find some traffic related to their service and automatically generates python snippets to replicate attacks. +## Disclaimer +The following disclaimer is intended to clarify the usage and purpose of any tools developed by Team Europe towards ICC. + +Please read the following statement carefully: + +Purpose: The tools developed by Team Europe members and coaches on this repository are intended for private use during ICC 2023 and ICC 2024. The tools are designed to enhance the team's performance, strategy, and overall participation in ICC. + +Private Use: The tools developed towards ICC are not intended for public distribution, or any use beyond the scope of ICC without explicit authorization from ENISA and the team members. + +Intellectual Property: The intellectual property rights of the tools developed by Team Europe towards ICC belong solely to the team and its members. Any unauthorized replication, modification, distribution, or use of the tools outside the specified competition is strictly prohibited. + +Exceptions: Upon agreement of ENISA and the Team Europe developers of the tools, authorisation can be provided for the private use of the same tools by national European Teams during the ECSC competition. + +By developing and using the tools developed by and for Team Europe towards ICC, you signify your understanding, acceptance, and compliance with the terms outlined in this disclaimer. + +--- + ## Origins Tulip was developed by Team Europe for use in the first International Cyber Security Challenge. The project is a fork of [flower](https://github.com/secgroup/flower), but it contains quite some changes: * New front-end (typescript / react / tailwind) diff --git a/docker-compose.yml b/docker-compose.yml index 842dc92..19be2b5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,12 +1,23 @@ -version: "3.2" +version: "3.5" services: - mongo: - image: mongo:5 + timescale: + build: services/timescale + image: tulip-timescale:latest + restart: unless-stopped + volumes: + - timescale-data:/var/lib/postgresql/data + - ./services/schema/system.sql:/docker-entrypoint-initdb.d/100_system.sql:ro + - ./services/schema/functions.sql:/docker-entrypoint-initdb.d/101_functions.sql:ro + - ./services/schema/schema.sql:/docker-entrypoint-initdb.d/102_schema.sql:ro networks: - internal - restart: always - ports: - - "27017:27017" + environment: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_USER: tulip + POSTGRES_DB: tulip + # This does not need to be adjusted, unless you actually want to limit it + # Postgres uses shared memory for caching, and docker assigns just 64 MB by default + shm_size: '128g' frontend: build: @@ -16,13 +27,16 @@ services: restart: unless-stopped ports: - "3000:3000" + expose: + - 3000 depends_on: - - mongo + - timescale - api networks: - internal environment: API_SERVER_ENDPOINT: http://api:5000/ + VIRTUAL_HOST: tulip.h4xx.eu api: build: @@ -31,13 +45,13 @@ services: image: tulip-api:latest restart: unless-stopped depends_on: - - mongo + - timescale networks: - internal volumes: - - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro + - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro,z environment: - TULIP_MONGO: ${TULIP_MONGO} + TIMESCALE: ${TIMESCALE} TULIP_TRAFFIC_DIR: ${TRAFFIC_DIR_DOCKER} FLAG_REGEX: ${FLAG_REGEX} TICK_START: ${TICK_START} @@ -45,21 +59,23 @@ services: VM_IP: ${VM_IP} flagids: - restart: on-failure + restart: unless-stopped build: context: services/flagids image: tulip-flagids:latest depends_on: - - mongo + - timescale networks: - internal environment: - TULIP_MONGO: ${TULIP_MONGO} + TIMESCALE: ${TIMESCALE} TICK_START: ${TICK_START} TICK_LENGTH: ${TICK_LENGTH} FLAGID_SCRAPE: ${FLAGID_SCRAPE} TEAM_ID: ${TEAM_ID} FLAGID_ENDPOINT: ${FLAGID_ENDPOINT} + VISUALIZER_URL: ${VISUALIZER_URL} + DUMP_PCAPS: ${DUMP_PCAPS} assembler: build: @@ -68,23 +84,35 @@ services: image: tulip-assembler:latest restart: unless-stopped depends_on: - - mongo + - timescale networks: - internal volumes: - - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro - command: "./assembler -dir ${TRAFFIC_DIR_DOCKER}" + - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro,z + # Command line flags most likely to fix a tulip issue: + # - -http-session-tracking: enable HTTP session tracking + # - -dir: directory to read traffic from + # - -skipchecksum: skip checksum validation + # - -flush-after: i.e. 2m Not needed in pcap rotation mode + # - -disable-converters: disable converters + # - -discard-extra-data: dont split large flow items, just discard them + command: "./assembler -http-session-tracking -skipchecksum -disable-converters -dir ${TRAFFIC_DIR_DOCKER}" environment: - TULIP_MONGO: ${TULIP_MONGO} + TIMESCALE: ${TIMESCALE} FLAG_REGEX: ${FLAG_REGEX} + TICK_START: ${TICK_START} TICK_LENGTH: ${TICK_LENGTH} FLAGID_SCAN: ${FLAGID_SCAN} FLAG_LIFETIME: ${FLAG_LIFETIME} + FLAG_VALIDATOR_TYPE: ${FLAG_VALIDATOR_TYPE} + FLAG_VALIDATOR_TEAM: ${FLAG_VALIDATOR_TEAM} PCAP_OVER_IP: ${PCAP_OVER_IP} + DUMP_PCAPS: ${DUMP_PCAPS} + DUMP_PCAPS_INTERVAL: ${DUMP_PCAPS_INTERVAL} + DUMP_PCAPS_FILENAME: ${DUMP_PCAPS_FILENAME} extra_hosts: - "host.docker.internal:host-gateway" - enricher: build: context: services/go-importer @@ -92,14 +120,17 @@ services: image: tulip-enricher:latest restart: unless-stopped depends_on: - - mongo + - timescale networks: - internal volumes: - - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro + - ${TRAFFIC_DIR_HOST}:${TRAFFIC_DIR_DOCKER}:ro,z command: "./enricher -eve ${TRAFFIC_DIR_DOCKER}/eve.json" environment: - TULIP_MONGO: ${TULIP_MONGO} + TIMESCALE: ${TIMESCALE} + +volumes: + timescale-data: networks: internal: diff --git a/frontend/Dockerfile-frontend b/frontend/Dockerfile-frontend index fbfc137..da6b849 100644 --- a/frontend/Dockerfile-frontend +++ b/frontend/Dockerfile-frontend @@ -13,4 +13,4 @@ RUN yarn run build EXPOSE 3000 -CMD yarn run preview --host --port 3000 +CMD ["yarn", "run", "preview", "--host", "--port", "3000"] diff --git a/frontend/src/api.ts b/frontend/src/api.ts index 8a593df..0dd27ab 100644 --- a/frontend/src/api.ts +++ b/frontend/src/api.ts @@ -4,12 +4,23 @@ import { API_BASE_PATH } from "./const"; import { Service, FullFlow, - Signature, TickInfo, Flow, FlowsQuery, + StatsQuery, + Stats, + TicksAttackInfo, + TicksAttackQuery, } from "./types"; +function base64DecodeUnicode(str: string) : string { + const text = atob(str); + const bytes = new Uint8Array(text.length); + for(let i = 0; i < text.length; i++) + bytes[i] = text.charCodeAt(i); + return new TextDecoder().decode(bytes); +} + export const tulipApi = createApi({ baseQuery: fetchBaseQuery({ baseUrl: API_BASE_PATH }), endpoints: (builder) => ({ @@ -21,6 +32,41 @@ export const tulipApi = createApi({ }), getFlow: builder.query({ query: (id) => `/flow/${id}`, + transformResponse: (flow: any): FullFlow => { + const representations: any = {}; + + for(const item of flow.items) { + if(!(item.kind in representations)) + representations[item.kind] = { type: item.kind, flow: [] }; + representations[item.kind].flow.push({ + from: item.direction, + data: base64DecodeUnicode(item.data), + b64: item.data, + time: new Date(item.time).getTime(), + }); + } + + return { + id: flow.id, + src_port: flow.port_src, + dst_port: flow.port_dst, + src_ip: flow.ip_src, + dst_ip: flow.ip_dst, + time: new Date(flow.time).getTime(), + duration: +(flow.duration * 1000).toFixed(0), + num_packets: flow.packets_count, + parent_id: flow.link_parent_id, + child_id: flow.link_child_id, + tags: flow.tags, + flags: flow.flags, + flagids: flow.flagids, + filename: flow.pcap_name, + service_tag: "", + suricata: [], + signatures: flow.signatures, + flow: Object.values(representations), + }; + }, }), getFlows: builder.query({ query: (query) => ({ @@ -30,14 +76,43 @@ export const tulipApi = createApi({ Accept: "application/json", "Content-Type": "application/json", }, - // TODO: fix the below tags mutation (make backend handle empty tags!) - // Diederik gives you a beer once this has been fixed - body: JSON.stringify({ - ...query, - includeTags: query.includeTags.length > 0 ? query.includeTags : undefined, - excludeTags: query.excludeTags.length > 0 ? query.excludeTags : undefined, - }), + body: query, }), + transformResponse: (response: Array) => { + return response.map((flow: any): Flow => ({ + id: flow.id, + src_port: flow.port_src, + dst_port: flow.port_dst, + src_ip: flow.ip_src, + dst_ip: flow.ip_dst, + time: new Date(flow.time).getTime(), + duration: +(flow.duration * 1000).toFixed(0), + num_packets: flow.packets_count, + parent_id: flow.link_parent_id, + child_id: flow.link_child_id, + tags: flow.tags, + flags: flow.flags, + flagids: flow.flagids, + filename: flow.pcap_name, + service_tag: "", + suricata: [], + })); + }, + }), + getStats: builder.query({ + query: (query) => ({ + url: `/stats`, + method: "GET", + headers: { + Accept: "application/json", + "Content-Type": "application/json", + }, + params: { + service: query.service, + tick_from: query.tick_from, + tick_to: query.tick_to, + } + }) }), getTags: builder.query({ query: () => `/tags`, @@ -45,20 +120,26 @@ export const tulipApi = createApi({ getTickInfo: builder.query({ query: () => `/tick_info`, }), - getSignature: builder.query({ - query: (id) => `/signature/${id}`, + getUnderAttack: builder.query({ + query: (query) => ({ + url: '/under_attack', + params: { + from_tick: query.from_tick, + to_tick: query.to_tick, + } + }), }), toPwnTools: builder.query({ query: (id) => ({ url: `/to_pwn/${id}`, responseHandler: "text" }), }), toSinglePythonRequest: builder.query< string, - { body: string; id: string; tokenize: boolean } + { body: string; id: string; item_index: number; tokenize: boolean } >({ - query: ({ body, id, tokenize }) => ({ + query: ({ body, id, item_index, tokenize }) => ({ url: `/to_single_python_request?tokenize=${ tokenize ? "1" : "0" - }&id=${id}`, + }&id=${id}&index=${item_index}`, method: "POST", responseHandler: "text", headers: { @@ -74,7 +155,15 @@ export const tulipApi = createApi({ }), }), starFlow: builder.mutation({ - query: ({ id, star }) => `/star/${id}/${star ? "1" : "0"}`, + query: ({ id, star }) => ({ + url: `/star`, + method: "POST", + headers: { + Accept: "application/json", + "Content-Type": "application/json", + }, + body: { id, star }, + }), // TODO: optimistic cache update // async onQueryStarted({ id, star }, { dispatch, queryFulfilled }) { @@ -110,10 +199,11 @@ export const { useGetFlowsQuery, useLazyGetFlowsQuery, useGetTagsQuery, - useGetSignatureQuery, useGetTickInfoQuery, useLazyToPwnToolsQuery, useLazyToFullPythonRequestQuery, useToSinglePythonRequestQuery, useStarFlowMutation, + useGetStatsQuery, + useGetUnderAttackQuery, } = tulipApi; diff --git a/frontend/src/components/Corrie.tsx b/frontend/src/components/Corrie.tsx index e943a26..077fbc2 100644 --- a/frontend/src/components/Corrie.tsx +++ b/frontend/src/components/Corrie.tsx @@ -1,6 +1,6 @@ -import { useSearchParams, useParams, useNavigate } from "react-router-dom"; +import { useSearchParams, useNavigate } from "react-router-dom"; import { useCallback } from "react"; -import { Flow } from "../types"; +import { Flow, Stats, TicksAttackInfo } from "../types"; import { SERVICE_FILTER_KEY, TEXT_FILTER_KEY, @@ -8,29 +8,47 @@ import { END_FILTER_KEY, CORRELATION_MODE_KEY, FLOW_LIST_REFETCH_INTERVAL_MS, + UNDER_ATTACK_REFETCH_INTERVAL_MS, } from "../const"; import useDebounce from "../hooks/useDebounce"; import ReactApexChart from "react-apexcharts"; import { ApexOptions } from "apexcharts"; -import { useGetFlowsQuery, useGetServicesQuery } from "../api"; +import { + useGetFlowsQuery, + useGetServicesQuery, + useGetStatsQuery, + useGetUnderAttackQuery +} from "../api"; +import { getTickStuff } from "../tick"; import { useAppSelector } from "../store"; +import { tagToColor } from "./Tag"; +interface TickInfoData { + startTick: number; + endTick: number; + flagLifetime: number; + tickToUnixTime: (a: number) => number; + unixTimeToTick: (a: number) => number; +} interface GraphProps { flowList: Flow[]; + statsList: Stats[]; + underAttackData: TicksAttackInfo; mode: string; searchParams: URLSearchParams; setSearchParams: (a: URLSearchParams) => void; onClickNavigate: (a: string) => void; + tickInfoData: TickInfoData } export const Corrie = () => { const { data: services } = useGetServicesQuery(); const includeTags = useAppSelector((state) => state.filter.includeTags); const excludeTags = useAppSelector((state) => state.filter.excludeTags); - const filterTags = useAppSelector((state) => state.filter.filterTags); const filterFlags = useAppSelector((state) => state.filter.filterFlags); const filterFlagids = useAppSelector((state) => state.filter.filterFlagids); + const tagIntersectionMode = useAppSelector((state) => state.filter.tagIntersectionMode); const [searchParams, setSearchParams] = useSearchParams(); @@ -43,17 +61,52 @@ export const Corrie = () => { const debounced_text_filter = useDebounce(text_filter, 300); - const { data: flowData, isLoading } = useGetFlowsQuery( + const mode = searchParams.get("correlation") ?? "time"; + const setCorrelationMode = (mode: string) => { + searchParams.set(CORRELATION_MODE_KEY, mode); + setSearchParams(searchParams); + }; + + const inactiveButtonClass = "bg-blue-100 text-gray-800 rounded-md px-2 py-1"; + const activeButtonClass = `${inactiveButtonClass} ring-2 ring-gray-500`; + + const navigate = useNavigate(); + const onClickNavigate = useCallback( + (loc: string) => navigate(loc, { replace: true }), + [navigate] + ); + + let { currentTick, flagLifetime, startTickParam, endTickParam, unixTimeToTick, tickToUnixTime } = getTickStuff(); + + let startTick = startTickParam ?? 0; + let endTick = endTickParam ?? currentTick; + if (startTick < 0) { + startTick = 0; + } + if (endTick < startTick) { + endTick = startTick; + } + + const needsStats = mode == "flags" || mode == "tags"; + + const statsData = needsStats ? useGetStatsQuery( + { + service: service_name, + tick_from: startTick, + tick_to: endTick, + } + ).data : []; + + const flowData = !needsStats ? useGetFlowsQuery( { - "flow.data": debounced_text_filter, - dst_ip: service?.ip, - dst_port: service?.port, - from_time: from_filter, - to_time: to_filter, - service: "", // FIXME - includeTags: includeTags, - excludeTags: excludeTags, - tags: filterTags, + regex_insensitive: debounced_text_filter, + ip_dst: service?.ip, + port_dst: service?.port, + time_from: from_filter ? new Date(parseInt(from_filter)).toISOString() : undefined, + time_to: to_filter ? new Date(parseInt(to_filter)).toISOString() : undefined, + tags_include: includeTags, + tags_exclude: excludeTags, + tag_intersection_mode: tagIntersectionMode, flags: filterFlags, flagids: filterFlagids, }, @@ -61,7 +114,18 @@ export const Corrie = () => { refetchOnMountOrArgChange: true, pollingInterval: FLOW_LIST_REFETCH_INTERVAL_MS, } - ); + ).data : []; + + // TODO: this fetches under attack data always - not sure how to fetch it only in under-attack mode due to react hooks having to be called in same order always + const underAttackData = useGetUnderAttackQuery( + { + from_tick: startTick, + to_tick: endTick + flagLifetime, + }, + { + pollingInterval: UNDER_ATTACK_REFETCH_INTERVAL_MS, + } + ).data; // TODO: fix the below transformation - move it to server // Diederik gives you a beer once it has been fixed @@ -72,27 +136,15 @@ export const Corrie = () => { ?.name ?? "unknown", })); - const mode = searchParams.get("correlation") ?? "time"; - const setCorrelationMode = (mode: string) => { - searchParams.set(CORRELATION_MODE_KEY, mode); - setSearchParams(searchParams); - }; - - const inactiveButtonClass = "bg-blue-100 text-gray-800 rounded-md px-2 py-1"; - const activeButtonClass = `${inactiveButtonClass} ring-2 ring-gray-500`; - - const navigate = useNavigate(); - const onClickNavigate = useCallback( - (loc: string) => navigate(loc, { replace: true }), - [navigate] - ); - const graphProps: GraphProps = { flowList: transformedFlowData || [], + statsList: statsData || [], + underAttackData: underAttackData || {}, mode: mode, searchParams: searchParams, setSearchParams: setSearchParams, onClickNavigate: onClickNavigate, + tickInfoData: { startTick, endTick, flagLifetime, unixTimeToTick, tickToUnixTime }, }; return ( @@ -120,16 +172,159 @@ export const Corrie = () => { > volume + + +
{(mode == "packets" || mode == "time") && TimePacketGraph(graphProps)} {mode == "volume" && VolumeGraph(graphProps)} + {(mode == "tags" || mode == "flags") && BarPerTickGraph(graphProps, mode)} + {(mode == "under-attack") && UnderAttackGraph(graphProps)}
); }; +function BarPerTickGraph(graphProps: GraphProps, mode: string) { + const statsList = graphProps.statsList; + const searchParams = graphProps.searchParams; + const setSearchParams = graphProps.setSearchParams; + let startTick = graphProps.tickInfoData.startTick; + let endTick = graphProps.tickInfoData.endTick; + let tickToUnixTime = graphProps.tickInfoData.tickToUnixTime; + + const SEARCH_CAP = 50; + const DEFAULT_CAP = 15; + + // Hard limit for performance reasons + if (searchParams.has(START_FILTER_KEY) && searchParams.has(END_FILTER_KEY)) { + startTick = Math.max(Math.max(0, startTick), endTick - SEARCH_CAP); + } else if (endTick - startTick > DEFAULT_CAP) { + startTick = Math.max(0, endTick - DEFAULT_CAP); + } + + var options: ApexOptions = { + plotOptions: { + bar: { + horizontal: false, + columnWidth: "90%", + } + }, + grid: { + position: "back", + xaxis: { + lines: { + show: endTick !== startTick + 1 + } + }, + yaxis: { + lines: { + show: false + } + } + }, + dataLabels: { + enabled: false, + }, + stroke: { + show: true, + width: 2, + colors: ['transparent'] + }, + xaxis: { + categories: Array.from({ length: endTick - startTick }, (_, i) => startTick + i), + title: { + text: "Ticks" + } + }, + yaxis: { + title: { + text: "Number of flows" + } + }, + tooltip: { + x: { + formatter: function (v) { + return "Tick " + v; + } + } + }, + chart: { + animations: { + enabled: false + }, + events: { + click: function (e, chartContext, options) { + const tick = options.dataPointIndex; + if (tick !== -1) { + const start = Math.floor(tickToUnixTime(tick + startTick)); + const end = Math.ceil(tickToUnixTime(tick + startTick + 1)); + searchParams.set(START_FILTER_KEY, start.toString()); + searchParams.set(END_FILTER_KEY, end.toString()); + setSearchParams(searchParams); + } + }, + }, + }, + }; + + let series: ApexAxisChartSeries = []; + + const colors: any = { + "tag_flag_in": tagToColor("flag-in"), + "tag_flag_out": tagToColor("flag-out"), + "tag_enemy": tagToColor("enemy"), + "tag_blocked": tagToColor("blocked"), + "tag_suricata": tagToColor("suricata"), + + "flag_in": tagToColor("flag-in"), + "flag_out": tagToColor("flag-out"), + }; + + Object.keys(colors).forEach(t => { + if ((mode == "tags" && t.startsWith("tag_")) || (mode == "flags" && t.startsWith("flag_"))) { + const data = Array(endTick - startTick).fill(0); + + statsList.forEach(s => { + data[s.tick - startTick] = s[t]; + }); + + series.push({ + name: t, + data: data, + color: colors[t] + }); + } + }); + + return ( + + ); +} + function TimePacketGraph(graphProps: GraphProps) { const flowList = graphProps.flowList; const mode = graphProps.mode; @@ -170,7 +365,7 @@ function TimePacketGraph(graphProps: GraphProps) { type: "datetime", // FIXME: Timezone is not displayed correctly }, labels: flowList.map((flow) => { - return flow._id.$oid; + return flow.id; }), chart: { animations: { @@ -260,7 +455,7 @@ function VolumeGraph(graphProps: GraphProps) { type: "datetime", // FIXME: Timezone is not displayed correctly }, labels: flowList.map((flow) => { - return flow._id.$oid; + return flow.id; }), chart: { animations: { @@ -280,3 +475,118 @@ function VolumeGraph(graphProps: GraphProps) { return ; } + +function UnderAttackGraph(graphProps: GraphProps) { + const underAttackData = graphProps.underAttackData; + const tickInfoData = graphProps.tickInfoData; + const tickToUnixTime = tickInfoData.tickToUnixTime; + const searchParams = graphProps.searchParams; + const setSearchParams = graphProps.setSearchParams; + + const options: ApexOptions = { + plotOptions: { + bar: { + horizontal: true, + barHeight: '30%', + rangeBarGroupRows: true, + }, + }, + tooltip: { + custom: (opts) => { + if (opts.y1 === opts.y2 - 1) return `Tick ${opts.y1}`; + + return `Ticks ${opts.y1} - ${opts.y2 - 1}`; + }, + }, + legend: { + show: false, + }, + xaxis: { + min: tickInfoData.startTick, + max: tickInfoData.endTick, + tickAmount: Math.min(Math.abs(tickInfoData.startTick - tickInfoData.endTick), 25), + decimalsInFloat: 0, + title: { + text: "Tick", + }, + }, + yaxis: { + tickAmount: 0, + }, + chart: { + animations: { + enabled: false, + }, + events: { + dataPointSelection: (event, ctx, config) => { + const y = config.w.config.series[config.seriesIndex].data[0].y; + const start = Math.floor(tickToUnixTime(y[0])); + const end = Math.ceil(tickToUnixTime(y[1])); + searchParams.set(START_FILTER_KEY, start.toString()); + searchParams.set(END_FILTER_KEY, end.toString()); + setSearchParams(searchParams); + }, + beforeZoom: function (chartContext, { xaxis }) { + const start = Math.floor(tickToUnixTime(xaxis.min)); + const end = Math.ceil(tickToUnixTime(xaxis.max)); + searchParams.set(START_FILTER_KEY, start.toString()); + searchParams.set(END_FILTER_KEY, end.toString()); + setSearchParams(searchParams); + }, + }, + } + }; + + // TODO: service names between visualizer and tulip don't necessarily match, how should we consider filters? + const ranges: Record = {}; + const lastSeen: Record = {}; + for (const tick in underAttackData) { + const tickNumber = Number(tick); + let from_tick = Math.max(0, tickNumber - (tickInfoData.flagLifetime - 1)); + + const services = underAttackData[tick]; + for (const service in services) { + const value = services[service]; + if (value <= 0) continue; + + ranges[service] = ranges[service] || []; + + // Heuristic: if we had previous ticks where we lost the flags, most likely an attack occured afterward + if (lastSeen[service] !== undefined) from_tick = Math.max(from_tick, lastSeen[service]!); + + ranges[service].push({ + from_tick: from_tick, + to_tick: tickNumber + 1, + }) + lastSeen[service] = tickNumber + 1; + } + } + + const series: ApexAxisChartSeries = []; + for (const service in ranges) { + for (const range of ranges[service]) { + series.push({ + data: [ + { + x: service, + y: [range.from_tick, range.to_tick], + goals: [ + { + name: 'tick start', + value: range.to_tick - 1, + strokeColor: '#CD2F2A', + }, + { + name: 'tick end', + value: range.to_tick, + strokeColor: '#CD2F2A', + } + ], + }, + ], + }) + } + } + + return ; +} diff --git a/frontend/src/components/FlowList.tsx b/frontend/src/components/FlowList.tsx index b20272a..a856aeb 100644 --- a/frontend/src/components/FlowList.tsx +++ b/frontend/src/components/FlowList.tsx @@ -6,6 +6,7 @@ import { } from "react-router-dom"; import { useState, useRef, useEffect } from "react"; import { useHotkeys } from 'react-hotkeys-hook'; +import { FetchBaseQueryError } from '@reduxjs/toolkit/query' import { Flow } from "../types"; import { SERVICE_FILTER_KEY, @@ -15,7 +16,7 @@ import { FLOW_LIST_REFETCH_INTERVAL_MS, } from "../const"; import { useAppSelector, useAppDispatch } from "../store"; -import { toggleFilterTag } from "../store/filter"; +import { toggleFilterTag, toggleTagIntersectMode } from "../store/filter"; import { HeartIcon, FilterIcon, LinkIcon } from "@heroicons/react/solid"; import { HeartIcon as EmptyHeartIcon } from "@heroicons/react/outline"; @@ -43,11 +44,11 @@ export function FlowList() { const { data: availableTags } = useGetTagsQuery(); const { data: services } = useGetServicesQuery(); - const filterTags = useAppSelector((state) => state.filter.filterTags); const filterFlags = useAppSelector((state) => state.filter.filterFlags); const filterFlagids = useAppSelector((state) => state.filter.filterFlagids); const includeTags = useAppSelector((state) => state.filter.includeTags); const excludeTags = useAppSelector((state) => state.filter.excludeTags); + const tagIntersectionMode = useAppSelector((state) => state.filter.tagIntersectionMode); const dispatch = useAppDispatch(); @@ -66,19 +67,22 @@ export function FlowList() { const debounced_text_filter = useDebounce(text_filter, 300); - const { data: flowData, isLoading, refetch } = useGetFlowsQuery( + const { + data: flowData, error: flowQueryError, + isLoading, isFetching, refetch, + startedTimeStamp, fulfilledTimeStamp, + } = useGetFlowsQuery( { - "flow.data": debounced_text_filter, - dst_ip: service?.ip, - dst_port: service?.port, - from_time: from_filter, - to_time: to_filter, - service: "", // FIXME - tags: filterTags, + regex_insensitive: debounced_text_filter, + ip_dst: service?.ip, + port_dst: service?.port, + time_from: from_filter ? new Date(parseInt(from_filter)).toISOString() : undefined, + time_to: to_filter ? new Date(parseInt(to_filter)).toISOString() : undefined, + tags_include: includeTags, + tags_exclude: excludeTags, + tag_intersection_mode: tagIntersectionMode, flags: filterFlags, flagids: filterFlagids, - includeTags: includeTags, - excludeTags: excludeTags }, { refetchOnMountOrArgChange: true, @@ -86,6 +90,23 @@ export function FlowList() { } ); + interface FlowQueryError { error: string } + const isFetchBaseQueryError = (error: unknown): error is FetchBaseQueryError => + typeof error === 'object' && error != null && 'status' in error + const isFlowQueryError = (error: unknown): error is FlowQueryError => + typeof error === 'object' && error != null && 'error' in error + const flowQueryErrorMessage = isFetchBaseQueryError(flowQueryError) + && isFlowQueryError(flowQueryError.data) + ? flowQueryError.data.error : null; + + let searchMessage = null; + if(isFetching) + searchMessage = "Searching..."; + else if(flowQueryErrorMessage) + searchMessage = `Error: ${flowQueryErrorMessage}`; + else if(startedTimeStamp && fulfilledTimeStamp) + searchMessage = `Search took ${fulfilledTimeStamp - startedTimeStamp}ms` + // TODO: fix the below transformation - move it to server // Diederik gives you a beer once it has been fixed const transformedFlowData = flowData?.map((flow) => ({ @@ -96,7 +117,7 @@ export function FlowList() { })); const onHeartHandler = async (flow: Flow) => { - await starFlow({ id: flow._id.$oid, star: !flow.tags.includes("starred") }); + await starFlow({ id: flow.id, star: !flow.tags.includes("starred") }); }; const navigate = useNavigate(); @@ -107,7 +128,7 @@ export function FlowList() { behavior: 'auto', done: () => { if (transformedFlowData && transformedFlowData[flowIndex ?? 0]) { - let idAtIndex = transformedFlowData[flowIndex ?? 0]._id.$oid; + let idAtIndex = transformedFlowData[flowIndex ?? 0].id; // if the current flow ID at the index indeed did change (ie because of keyboard navigation), we need to update the URL as well as local ID if (idAtIndex !== openedFlowID) { navigate(`/flow/${idAtIndex}?${searchParams}`) @@ -130,7 +151,7 @@ export function FlowList() { setTransformedFlowDataLength(transformedFlowData?.length) for (let i = 0; i < transformedFlowData?.length; i++) { - if (transformedFlowData[i]._id.$oid === openedFlowID) { + if (transformedFlowData[i].id === openedFlowID) { if (i !== flowIndex) { setFlowIndex(i) } @@ -181,9 +202,17 @@ export function FlowList() { {showFilters && (
-

- Intersection filter -

+
+

+ Intersection filter +

+ +
{(availableTags ?? []).map((tag) => (
+ { searchMessage &&
{searchMessage}
} ( setFlowIndex(index)} - key={flow._id.$oid} + key={flow.id} className="focus-visible:rounded-md" //style={{ paddingTop: '1em' }} > @@ -239,7 +269,8 @@ function FlowListEntry({ flow, isActive, onHeartClick }: FlowListEntryProps) { const formatted_time_h_m_s = format(new Date(flow.time), "HH:mm:ss"); const formatted_time_ms = format(new Date(flow.time), ".SSS"); - const isStarred = flow.tags.includes("starred"); + const [isStarred, setStarred] = useState(flow.tags.includes("starred")); + // Filter tag list for tags that are handled specially const filtered_tag_list = flow.tags.filter((t) => t != "starred"); @@ -259,10 +290,11 @@ function FlowListEntry({ flow, isActive, onHeartClick }: FlowListEntryProps) {
{ + setStarred(!isStarred); onHeartClick(flow); }} > - {flow.tags.includes("starred") ? ( + {isStarred ? ( ) : ( @@ -270,8 +302,7 @@ function FlowListEntry({ flow, isActive, onHeartClick }: FlowListEntryProps) {
- {flow.child_id.$oid != "000000000000000000000000" || - flow.parent_id.$oid != "000000000000000000000000" ? ( + {flow.child_id != null || flow.parent_id != null ? ( ) : undefined}
diff --git a/frontend/src/components/Header.tsx b/frontend/src/components/Header.tsx index 8c5b010..531df08 100644 --- a/frontend/src/components/Header.tsx +++ b/frontend/src/components/Header.tsx @@ -17,13 +17,13 @@ import { FIRST_DIFF_KEY, SECOND_DIFF_KEY, SERVICE_REFETCH_INTERVAL_MS, - TICK_REFETCH_INTERVAL_MS, + REPR_ID_KEY, } from "../const"; import { useGetFlowQuery, useGetServicesQuery, - useGetTickInfoQuery, } from "../api"; +import { getTickStuff } from "../tick"; function ServiceSelection() { const FILTER_KEY = SERVICE_FILTER_KEY; @@ -95,85 +95,9 @@ function TextSearch() { ); } -function useMessyTimeStuff() { - let [searchParams, setSearchParams] = useSearchParams(); - - const { data: tickInfoData } = useGetTickInfoQuery(undefined, { - pollingInterval: TICK_REFETCH_INTERVAL_MS, - }); - - // TODO: prevent having to work with default values here - let startDate = "1970-01-01T00:00:00Z"; - let tickLength = 1000; - - if (tickInfoData) { - startDate = tickInfoData.startDate; - tickLength = tickInfoData.tickLength; - } - - function setTimeParam(startTick: string, param: string) { - const parsedTick = startTick === "" ? undefined : parseInt(startTick); - const unixTime = tickToUnixTime(parsedTick); - if (unixTime) { - searchParams.set(param, unixTime.toString()); - } else { - searchParams.delete(param); - } - setSearchParams(searchParams); - } - - const startTimeParamUnix = searchParams.get(START_FILTER_KEY); - const endTimeParamUnix = searchParams.get(END_FILTER_KEY); - - function unixTimeToTick(unixTime: string | null): number | undefined { - if (unixTime === null) { - return; - } - let unixTimeInt = parseInt(unixTime); - if (isNaN(unixTimeInt)) { - return; - } - const tick = Math.floor( - (unixTimeInt - new Date(startDate).valueOf()) / tickLength - ); - - return tick; - } - - function tickToUnixTime(tick?: number) { - if (!tick) { - return; - } - const unixTime = new Date(startDate).valueOf() + tickLength * tick; - return unixTime; - } - - const startTick = unixTimeToTick(startTimeParamUnix); - const endTick = unixTimeToTick(endTimeParamUnix); - const currentTick = unixTimeToTick(new Date().valueOf().toString()); - - function setToLastnTicks(n: number) { - const startTick = (currentTick ?? 0) - n; - const endTick = (currentTick ?? 0) + 1; // to be sure - setTimeParam(startTick.toString(), START_FILTER_KEY); - setTimeParam(endTick.toString(), END_FILTER_KEY); - } - - return { - unixTimeToTick, - startDate, - tickLength, - setTimeParam, - startTick, - endTick, - currentTick, - setToLastnTicks, - }; -} function StartDateSelection() { - const { setTimeParam, startTick } = useMessyTimeStuff(); - + let { startTickParam, setTimeParam } = getTickStuff(); return (
{ - setTimeParam(event.target.value, START_FILTER_KEY); + setTimeParam(event.target.value == "" ? null : parseInt(event.target.value), START_FILTER_KEY); }} >
@@ -191,8 +115,7 @@ function StartDateSelection() { } function EndDateSelection() { - const { setTimeParam, endTick } = useMessyTimeStuff(); - + let { endTickParam, setTimeParam } = getTickStuff(); return (
{ - setTimeParam(event.target.value, END_FILTER_KEY); + setTimeParam(event.target.value == "" ? null : parseInt(event.target.value), END_FILTER_KEY); }} >
@@ -218,9 +141,11 @@ function FirstDiff() { function setFirstDiffFlow() { let textFilter = params.id; + let reprId = searchParams.get(REPR_ID_KEY); + let reprIdSlug = reprId ? `${textFilter}:${reprId}` : `${textFilter}` if (textFilter) { - searchParams.set(FIRST_DIFF_KEY, textFilter); - setFirstFlow(textFilter); + searchParams.set(FIRST_DIFF_KEY, reprIdSlug); + setFirstFlow(reprIdSlug); } else { searchParams.delete(FIRST_DIFF_KEY); setFirstFlow(""); @@ -259,9 +184,11 @@ function SecondDiff() { function setSecondDiffFlow() { let textFilter = params.id; + let reprId = searchParams.get(REPR_ID_KEY); + let reprIdSlug = reprId ? `${textFilter}:${reprId}` : `${textFilter}` if (textFilter) { - searchParams.set(SECOND_DIFF_KEY, textFilter); - setSecondFlow(textFilter); + searchParams.set(SECOND_DIFF_KEY, reprIdSlug); + setSecondFlow(reprIdSlug); } else { searchParams.delete(SECOND_DIFF_KEY); setSecondFlow(""); @@ -319,15 +246,15 @@ function Diff() { } export function Header() { + let { currentTick, setToLastnTicks, setTimeParam } = getTickStuff(); let [searchParams] = useSearchParams(); - const { setToLastnTicks, currentTick, setTimeParam } = useMessyTimeStuff(); useHotkeys('a', () => setToLastnTicks(5)); useHotkeys('c', () => { (document.getElementById("startdateselection") as HTMLInputElement).value = ""; (document.getElementById("enddateselection") as HTMLInputElement).value = ""; - setTimeParam("", START_FILTER_KEY); - setTimeParam("", END_FILTER_KEY); + setTimeParam(null, START_FILTER_KEY); + setTimeParam(null, END_FILTER_KEY); }); return ( diff --git a/frontend/src/const.ts b/frontend/src/const.ts index b2bea03..656b105 100644 --- a/frontend/src/const.ts +++ b/frontend/src/const.ts @@ -1,4 +1,4 @@ -export const API_BASE_PATH = "/api"; +export const API_BASE_PATH = `${window.location.origin}/api`; export const TEXT_FILTER_KEY = "text"; export const SERVICE_FILTER_KEY = "service"; @@ -6,9 +6,11 @@ export const START_FILTER_KEY = "start"; export const END_FILTER_KEY = "end"; export const FIRST_DIFF_KEY = "first"; export const SECOND_DIFF_KEY = "second"; +export const REPR_ID_KEY = "reprid"; export const CORRELATION_MODE_KEY = "correlation"; export const SERVICE_REFETCH_INTERVAL_MS = 15000; export const TICK_REFETCH_INTERVAL_MS = 10000; export const FLOW_LIST_REFETCH_INTERVAL_MS = 30000; +export const UNDER_ATTACK_REFETCH_INTERVAL_MS = 30000; export const MAX_LENGTH_FOR_HIGHLIGHT = 400000; diff --git a/frontend/src/pages/DiffView.tsx b/frontend/src/pages/DiffView.tsx index 70814c5..b4f8b4b 100644 --- a/frontend/src/pages/DiffView.tsx +++ b/frontend/src/pages/DiffView.tsx @@ -1,5 +1,6 @@ import { useSearchParams, Link, useParams } from "react-router-dom"; import { useState } from "react"; +import { Buffer } from "buffer"; import { FullFlow } from "../types"; @@ -56,8 +57,8 @@ const deriveDisplayMode = ( i++ ) { if ( - !isASCII(firstFlow.flow[i].data) || - !isASCII(secondFlow.flow[i].data) + !isASCII(firstFlow.flow[0].flow[i].data) || + !isASCII(secondFlow.flow[0].flow[i].data) ) { return displayOptions[1]; } @@ -69,16 +70,20 @@ const deriveDisplayMode = ( export function DiffView() { let [searchParams] = useSearchParams(); - const firstFlowId = searchParams.get(FIRST_DIFF_KEY); - const secondFlowId = searchParams.get(SECOND_DIFF_KEY); - - let { data: firstFlow, isLoading: firstFlowLoading } = useGetFlowQuery( + const firstFlowParam = searchParams.get(FIRST_DIFF_KEY); + const firstFlowId = firstFlowParam?.split(":")[0]; + const firstFlowRepr = parseInt(firstFlowParam?.split(":")[1] ?? "0"); + const secondFlowParam = searchParams.get(SECOND_DIFF_KEY); + const secondFlowId = secondFlowParam?.split(":")[0]; + const secondFlowRepr = parseInt(secondFlowParam?.split(":")[1] ?? "0"); + + let { data: firstFlow, isLoading: firstFlowLoading, isError: firstFlowError } = useGetFlowQuery( firstFlowId!, { skip: firstFlowId === null, } ); - let { data: secondFlow, isLoading: secondFlowLoading } = useGetFlowQuery( + let { data: secondFlow, isLoading: secondFlowLoading, isError: secondFlowError } = useGetFlowQuery( secondFlowId!, { skip: secondFlowId === null, @@ -89,7 +94,7 @@ export function DiffView() { deriveDisplayMode(firstFlow!, secondFlow!) ); - if (firstFlowId === null || secondFlowId === null) { + if (firstFlowError || secondFlowError) { return
Invalid flow id
; } @@ -113,9 +118,9 @@ export function DiffView() {
{Array.from( { - length: Math.min(firstFlow!.flow.length, secondFlow!.flow.length), + length: Math.min(firstFlow!.flow[firstFlowRepr].flow.length, secondFlow!.flow[secondFlowRepr].flow.length), }, - (_, i) => Flow(firstFlow!.flow[i].data, secondFlow!.flow[i].data) + (_, i) => Flow(firstFlow!.flow[firstFlowRepr].flow[i].data, secondFlow!.flow[secondFlowRepr].flow[i].data) )}
)} @@ -125,12 +130,12 @@ export function DiffView() {
{Array.from( { - length: Math.min(firstFlow!.flow.length, secondFlow!.flow.length), + length: Math.min(firstFlow!.flow[firstFlowRepr].flow.length, secondFlow!.flow[secondFlowRepr].flow.length), }, (_, i) => Flow( - hexy(firstFlow!.flow[i].data, { format: "twos" }), - hexy(secondFlow!.flow[i].data, { format: "twos" }) + hexy(Buffer.from(firstFlow!.flow[firstFlowRepr].flow[i].b64, 'base64'), { format: "twos" }), + hexy(Buffer.from(secondFlow!.flow[secondFlowRepr].flow[i].b64, 'base64'), { format: "twos" }) ) )}
diff --git a/frontend/src/pages/FlowView.tsx b/frontend/src/pages/FlowView.tsx index 1b5e42d..d06635d 100644 --- a/frontend/src/pages/FlowView.tsx +++ b/frontend/src/pages/FlowView.tsx @@ -1,12 +1,15 @@ import { useSearchParams, Link, useParams, useNavigate } from "react-router-dom"; import React, { ChangeEvent, useDeferredValue, useEffect, useState } from "react"; -import { useHotkeys } from 'react-hotkeys-hook'; +import { useHotkeys } from "react-hotkeys-hook"; import { FlowData, FullFlow } from "../types"; import { Buffer } from "buffer"; import { TEXT_FILTER_KEY, MAX_LENGTH_FOR_HIGHLIGHT, API_BASE_PATH, + REPR_ID_KEY, + FIRST_DIFF_KEY, + SECOND_DIFF_KEY, } from "../const"; import { ArrowCircleLeftIcon, @@ -14,6 +17,7 @@ import { ArrowCircleUpIcon, ArrowCircleDownIcon, DownloadIcon, + LightningBoltIcon, } from "@heroicons/react/solid"; import { format } from "date-fns"; @@ -22,12 +26,14 @@ import { useCopy } from "../hooks/useCopy"; import { RadioGroup } from "../components/RadioGroup"; import { useGetFlowQuery, + useGetServicesQuery, useLazyToFullPythonRequestQuery, useLazyToPwnToolsQuery, useToSinglePythonRequestQuery, useGetFlagRegexQuery, } from "../api"; -import escapeStringRegexp from 'escape-string-regexp'; +import { getTickStuff } from "../tick"; +import escapeStringRegexp from "escape-string-regexp"; const SECONDARY_NAVBAR_HEIGHT = 50; @@ -68,32 +74,83 @@ function FlowContainer({ } function HexFlow({ flow }: { flow: FlowData }) { - const hex = hexy(Buffer.from(flow.b64, 'base64'), { format: "twos" }); + const hex = hexy(Buffer.from(flow.b64, "base64"), { format: "twos" }); return {hex}; } function highlightText(flowText: string, search_string: string, flag_string: string) { - if (flowText.length > MAX_LENGTH_FOR_HIGHLIGHT || flag_string === '') { + if (flowText.length > MAX_LENGTH_FOR_HIGHLIGHT || (flag_string === "" && search_string === "")) { return flowText } try { - const flag_regex = new RegExp(`(${flag_string})`, 'g'); - const search_regex = new RegExp(`(${search_string})`, 'gi'); - const combined_regex = new RegExp(`${search_regex.source}|${flag_regex.source}`, 'gi'); - let parts; - if (search_string !== '') { - parts = flowText.split(combined_regex); - } else { - parts = flowText.split(flag_regex); + const searchClasses = "bg-orange-200 rounded-sm"; + const flagClasses = "bg-red-200 rounded-sm"; + + // Matches are stored as `[start index, end index]`. + // For some reason tsc compiler (during build) thinks that `x.index` can be undefined (no, it can't). + // I wasn't able to find a workaround for it so @ts-ignore it is... + // Other way would be `x.index ?? 0` but that seems like it is doing something more than fixing typescript issues. + // @ts-ignore + const flagMatches: [number, number][] = ( + flag_string === "" + ? [] + // @ts-ignore + : [...flowText.matchAll(new RegExp(flag_string, "g"))].map(x => [x.index, x.index + x[0].length]) + ); + // @ts-ignore + const searchMatches: [number, number][] = ( + search_string === "" + ? [] + // @ts-ignore + : [...flowText.matchAll(new RegExp(search_string, "gi"))].map(x => [x.index, x.index + x[0].length]) + ); + + let parts = []; + let currentIndex = 0, flagMatchIndex = 0, searchMatchIndex = 0; + while (true) { + // Pick next match + let isSearchMatch = null; + if (flagMatchIndex < flagMatches.length && searchMatchIndex < searchMatches.length) { + isSearchMatch = searchMatches[searchMatchIndex][0] <= flagMatches[flagMatchIndex][0]; + } else if (searchMatchIndex < searchMatches.length) { + isSearchMatch = true; + } else if (flagMatchIndex < flagMatches.length) { + isSearchMatch = false; + } + let match = ( + isSearchMatch === null + ? null + : isSearchMatch ? searchMatches[searchMatchIndex] : flagMatches[flagMatchIndex] + ); + + // Produce element for remaining text if there is no match + if (match === null) { + parts.push({flowText.slice(currentIndex)}); + break; + } + + // Produce element for part between previous and next/current match + if (currentIndex != match[0]) { + parts.push({flowText.slice(currentIndex, match[0])}); + } + + // Produce element for current match + parts.push({flowText.slice(match[0], match[1])}); + + // Advance position to end of match + currentIndex = match[1]; + + // Advance "pointers" for flag matches + while (flagMatchIndex < flagMatches.length && flagMatches[flagMatchIndex][1] <= currentIndex) flagMatchIndex++; + // If current match ends in the middle of next match, we cut that overlaping part out + if (flagMatchIndex < flagMatches.length && flagMatches[flagMatchIndex][0] < currentIndex) flagMatches[flagMatchIndex][0] = currentIndex; + // Do the same also for search matches + while (searchMatchIndex < searchMatches.length && searchMatches[searchMatchIndex][1] <= currentIndex) searchMatchIndex++; + if (searchMatchIndex < searchMatches.length && searchMatches[searchMatchIndex][0] < currentIndex) searchMatches[searchMatchIndex][0] = currentIndex; } - const searchClasses = "bg-orange-200 rounded-sm" - const flagClasses = "bg-red-200 rounded-sm" - return { parts.map((part, i) => - - { part } - ) - }; - } catch(error) { - console.log(error) + + return {parts}; + } catch (error) { + console.log(error); return flowText; } } @@ -102,7 +159,7 @@ function TextFlow({ flow }: { flow: FlowData }) { let [searchParams] = useSearchParams(); const text_filter = searchParams.get(TEXT_FILTER_KEY); const { data: flag_regex } = useGetFlagRegexQuery(); - const text = highlightText(flow.data, text_filter ?? '', flag_regex ?? ''); + const text = highlightText(flow.data, text_filter ?? "", flag_regex ?? ""); return {text}; } @@ -131,13 +188,16 @@ function WebFlow({ flow }: { flow: FlowData }) { function PythonRequestFlow({ full_flow, flow, + item_index, }: { full_flow: FullFlow; flow: FlowData; + item_index: number, }) { const { data } = useToSinglePythonRequestQuery({ body: flow.b64, - id: full_flow._id.$oid, + id: full_flow.id, + item_index, tokenize: true, }); @@ -147,6 +207,7 @@ function PythonRequestFlow({ interface FlowProps { full_flow: FullFlow; flow: FlowData; + flow_item_index: number; delta_time: number; id: string; } @@ -164,16 +225,18 @@ function getFlowBody(flow: FlowData, flowType: string) { if (flowType == "Web") { const contentType = flow.data.match(/Content-Type: ([^\s;]+)/im)?.[1]; if (contentType) { - const body = Buffer.from(flow.b64, 'base64').subarray(flow.data.indexOf('\r\n\r\n')+4); + const body = Buffer.from(flow.b64, "base64").subarray(flow.data.indexOf("\r\n\r\n") + 4); return [contentType, body] } } return null } -function Flow({ full_flow, flow, delta_time, id }: FlowProps) { +function Flow({ full_flow, flow, flow_item_index, delta_time, id }: FlowProps) { const formatted_time = format(new Date(flow.time), "HH:mm:ss:SSS"); - const displayOptions = ["Plain", "Hex", "Web", "PythonRequest"]; + const displayOptions = flow.from === "s" + ? ["Plain", "Hex", "Web"] + : ["Plain", "Hex", "PythonRequest"]; // Basic type detection, currently unused const [displayOption, setDisplayOption] = useState("Plain"); @@ -204,7 +267,7 @@ function Flow({ full_flow, flow, delta_time, id }: FlowProps) { onClick={async () => { window.open( "https://gchq.github.io/CyberChef/#input=" + - encodeURIComponent(flow.b64) + encodeURIComponent(flow.b64) ); }} > @@ -212,28 +275,28 @@ function Flow({ full_flow, flow, delta_time, id }: FlowProps) { {flowType == "Web" && flowBody && ( + className="bg-gray-200 py-1 px-2 rounded-md text-sm ml-2" + onClick={async () => { + window.open( + "https://gchq.github.io/CyberChef/#input=" + + encodeURIComponent(flowBody[1].toString("base64")) + ); + }} + > + Open body in CC + )} {flowType == "Web" && flowBody && ( + className="bg-gray-200 py-1 px-2 rounded-md text-sm ml-2" + onClick={async () => { + const blob = new Blob([flowBody[1]], { + type: flowBody[0].toString(), + }); + const url = window.URL.createObjectURL(blob); + const a = document.createElement("a"); + a.style.display = "none"; + a.href = url; + a.download = "tulip-dl-" + id + ".dat"; + document.body.appendChild(a); + a.click(); + window.URL.revokeObjectURL(url); + a.remove(); + }} + > + Download body + )} )}
@@ -299,8 +363,10 @@ function formatIP(ip: string) { } function FlowOverview({ flow }: { flow: FullFlow }) { - const FILTER_KEY = TEXT_FILTER_KEY; let [searchParams, setSearchParams] = useSearchParams(); + const { unixTimeToTick } = getTickStuff(); + const { data: services } = useGetServicesQuery(); + const service = services?.find((s) => s.ip === flow.dst_ip && s.port === flow.dst_port)?.name ?? "unknown"; return (
{flow.signatures?.length > 0 ? ( @@ -311,15 +377,15 @@ function FlowOverview({ flow }: { flow: FullFlow }) { return (
-
Message:
-
{sig.msg}
+
Message: 
+
{sig.message}
-
Rule ID:
+
Rule ID: 
{sig.id}
-
Action taken:
+
Action taken: 
Meta
-
Source:
-
- + -
-
Tags:
-
[{flow.tags.join(", ")}]
-
Flags:
-
- [{flow.flags.map((query, i) => ( - - {i > 0 ? ', ' : ''} - +
+ Tags:  + [{flow.tags.join(", ")}] +
+
+ Tick:  + {unixTimeToTick(flow.time)} +
+
+ Service:  + {service} +
+
+ Flags:  + + [{flow.flags.map((query, i) => ( + + {i > 0 ? ", " : ""} + + + ))}] - ))}]
-
Flagids:
-
- [{flow.flagids.map((query, i) => ( - - {i > 0 ? ', ' : ''} - - - ))}] + }} + > + {query} + + + ))}] +
-
-
Source - Target (Duration):
-
-
- {" "} - {formatIP(flow.src_ip)}: - {flow.src_port} -
-
-
-
- {formatIP(flow.dst_ip)}: - {flow.dst_port} -
-
+
+ Source - Target (Duration):  +
+
+ {formatIP(flow.src_ip)}: + {flow.src_port} +
+ - +
+ {formatIP(flow.dst_ip)}: + {flow.dst_port} +
({flow.duration} ms)
@@ -413,14 +488,16 @@ export function FlowView() { const id = params.id; + const [reprId, setReprId] = useState(parseInt(searchParams.get(REPR_ID_KEY) ?? "0")); + const { data: flow, isError, isLoading } = useGetFlowQuery(id!, { skip: id === undefined }); const [triggerPwnToolsQuery] = useLazyToPwnToolsQuery(); const [triggerFullPythonRequestQuery] = useLazyToFullPythonRequestQuery(); async function copyAsPwn() { - if (flow?._id.$oid) { - const { data } = await triggerPwnToolsQuery(flow?._id.$oid); + if (flow?.id) { + const { data } = await triggerPwnToolsQuery(flow?.id); console.log(data); return data || ""; } @@ -438,8 +515,8 @@ export function FlowView() { }); async function copyAsRequests() { - if (flow?._id.$oid) { - const { data } = await triggerFullPythonRequestQuery(flow?._id.$oid); + if (flow?.id) { + const { data } = await triggerFullPythonRequestQuery(flow?.id); return data || ""; } return ""; @@ -458,30 +535,61 @@ export function FlowView() { // TODO: account for user scrolling - update currentFlow accordingly const [currentFlow, setCurrentFlow] = useState(-1); - useHotkeys('h', () => { + useHotkeys("h", () => { // we do this for the scroll to top if (currentFlow === 0) { document.getElementById(`${id}-${currentFlow}`)?.scrollIntoView(true) } setCurrentFlow(fi => Math.max(0, fi - 1)) }, [currentFlow]); - useHotkeys('l', () => { - if (currentFlow === (flow?.flow?.length ?? 1)-1) { + useHotkeys("l", () => { + if (currentFlow === (flow?.flow[reprId]?.flow?.length ?? 1) - 1) { document.getElementById(`${id}-${currentFlow}`)?.scrollIntoView(true) } - setCurrentFlow(fi => Math.min((flow?.flow?.length ?? 1)-1, fi + 1)) - }, [currentFlow, flow?.flow?.length]); + setCurrentFlow(fi => Math.min((flow?.flow[reprId]?.flow?.length ?? 1) - 1, fi + 1)) + }, [currentFlow, flow?.flow[reprId]?.flow?.length, reprId]); useEffect( () => { if (currentFlow < 0) { return } - document.getElementById(`${id}`)?.scrollIntoView(true) + document.getElementById(`${id}-${currentFlow}`)?.scrollIntoView(true) }, [currentFlow] ) + useHotkeys("m", () => { + setReprId(ri => (ri + 1) % (flow?.flow.length ?? 1)) + }, [reprId, flow?.flow.length]); + + // when the reprId changes, we update the url + useEffect( + () => { + if (reprId === 0) { + searchParams.delete(REPR_ID_KEY) + setSearchParams(searchParams) + return + } + searchParams.set(REPR_ID_KEY, reprId.toString()); + setSearchParams(searchParams) + }, + [reprId] + ) + + // if the flow doesn't have the representation we're looking for, we fallback to raw + useEffect( + () => { + if (flow?.flow.length == undefined || flow?.flow.length === 0) { + return + } + if ((flow?.flow.length - 1) < reprId) { + setReprId(0) + } + }, + [flow?.flow.length] + ) + if (isError) { return
Error while fetching flow
; } @@ -496,39 +604,63 @@ export function FlowView() { className="sticky shadow-md top-0 bg-white overflow-auto border-b border-b-gray-200 flex" style={{ height: SECONDARY_NAVBAR_HEIGHT, zIndex: 100 }} > - {(flow?.child_id?.$oid != "000000000000000000000000" || flow?.parent_id?.$oid != "000000000000000000000000") ? ( -
+ {(flow?.child_id != null || flow?.parent_id != null) ? ( +
-
- ) : undefined} +
+ ) : undefined}
+

Decoders ({flow?.flow.length}):

+ + {reprId > 0 ? : undefined}