From b893cbb8f2fadfef0af1fcd990a449b71b17e3f0 Mon Sep 17 00:00:00 2001 From: Geert Vanderkelen Date: Tue, 19 Jan 2021 10:57:12 +0100 Subject: [PATCH] Change first run of TIE API request to use first_seen_at Previously, the first request to TIE was using the `updated_at_since` field to order data. This proved to be not so good, and also slow. We change the first request to use `first_seen_at` instead. --- bin/tie2index.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/bin/tie2index.py b/bin/tie2index.py index ef00b5e..bcf74f5 100644 --- a/bin/tie2index.py +++ b/bin/tie2index.py @@ -1,14 +1,13 @@ # Copyright (c) 2017, 2019, DCSO GmbH -import urllib2 -import json -import datetime import csv +import datetime +import json +import os import re import sys -import os -from cStringIO import StringIO -from sets import Set + +import urllib2 from splunk.clilib import cli_common as cli csv.field_size_limit(sys.maxsize) @@ -32,7 +31,7 @@ def first_run(): period = (datetime.date.today() - datetime.timedelta(30)).strftime('%Y-%m-%dT%H:%M:%SZ') - request = urllib2.Request("{}?updated_at_since={}&order_by=seq&direction=asc".format(TIE_API,period)) + request = urllib2.Request("{}?first_seen_at={}&order_by=seq&direction=asc".format(TIE_API,period)) request.add_header("X-Authorization", 'bearer {}'.format(TIE_TOKEN)) request.add_header("Accept", 'application/json') contents = json.loads(urllib2.urlopen(request).read()) @@ -66,9 +65,9 @@ def query_tie(seq): with open(os.path.join(os.path.dirname(__file__),'../local/seq.json'), 'r') as seqinput: seqfile = json.load(seqinput) else: - data = {'tie':{'seq_number': 0}} + seqfile = {'tie':{'seq_number': 0}} with open(os.path.join(os.path.dirname(__file__),'../local/seq.json'), 'w') as outfile: - json.dump(data,outfile) + json.dump(seqfile, outfile) tie_args = cli.getConfStanza('dcso_tie_setup','tie') filter_args = cli.getConfStanza('dcso_tie_setup','filter')