diff --git a/software/backend/lib/EP/Cache.pm b/software/backend/lib/EP/Cache.pm index d77b1a0..384413e 100644 --- a/software/backend/lib/EP/Cache.pm +++ b/software/backend/lib/EP/Cache.pm @@ -39,7 +39,7 @@ information. use Mojo::Base -base; use Carp; use DBI; -use Mojo::JSON; +use Mojo::JSON qw(decode_json encode_json); use Encode; use EP::Exception qw(mkerror); @@ -159,7 +159,6 @@ has 'dbhPe'; has encodeUtf8 => sub { find_encoding('utf8') }; has tree => sub { [] }; -has json => sub { Mojo::JSON->new }; =head2 B(I) @@ -324,7 +323,7 @@ sub add { } $self->log->debug("keygen $rawNodeId => $nodeId"); eval { - $dbc->do("INSERT INTO node (rowid,data) VALUES (?,?)",{},$nodeId,$self->json->encode($nodeData)); + $dbc->do("INSERT INTO node (rowid,data) VALUES (?,?)",{},$nodeId,encode_json($nodeData)); }; if ($@){ $self->log->warn("$@"); @@ -433,10 +432,9 @@ sub getNodes { my $dbh = $self->dbhCa; my $sth = $dbh->prepare("SELECT docid,data FROM node WHERE data MATCH ? LIMIT ? OFFSET ?"); $sth->execute($self->encodeUtf8->encode($expression),$limit,$offset); - my $json = $self->json; my @return; while (my $row = $sth->fetchrow_hashref){ - my $data = $json->decode($row->{data}); + my $data = decode_json($row->{data}); my $entry = { map { $_ => $data->{$_} } @{$self->searchCols} }; $entry->{__epId} = $row->{docid}; push @return, $entry; @@ -455,8 +453,7 @@ sub getNode { my $nodeId = shift; my $dbh = $self->dbhCa; my @row = $dbh->selectrow_array("SELECT data FROM node WHERE docid = ?",{},$nodeId); - my $json = $self->json; - my $ret = $json->decode($row[0]); + my $ret = decode_json($row[0]); $ret->{__epId} = $nodeId; return $ret; } @@ -493,7 +490,7 @@ sub getBranch { $branch->[1] =~ s/^{SORT:.+?}//; my @leaves; while (my ($docid,$row) = $sth->fetchrow_array()){ - my $data = $self->json->decode($row); + my $data = decode_json($row); $data->{__epId} = $docid; push @leaves, [ map { $data->{$_} } @{$self->treeCols} ]; } @@ -556,7 +553,7 @@ SQL_END lb => $row->{label}, # label login => $row->{login}, # owner - login private => $row->{private}, # private - cfg => $self->json->decode($row->{cf}), #cfg + cfg => decode_json($row->{cf}), #cfg mine => $row->{mine}, }; } @@ -583,7 +580,7 @@ Returns: sub saveDash { my $self = shift; - my $cfg = $self->json->encode(shift); + my $cfg = encode_json(shift); my $label = shift; my $id = shift; my $updateTime = shift; diff --git a/software/backend/lib/EP/Visualizer/TorrusAggregateBrowserChart.pm b/software/backend/lib/EP/Visualizer/TorrusAggregateBrowserChart.pm index 4e8bf20..fab5133 100644 --- a/software/backend/lib/EP/Visualizer/TorrusAggregateBrowserChart.pm +++ b/software/backend/lib/EP/Visualizer/TorrusAggregateBrowserChart.pm @@ -53,7 +53,8 @@ all the methods from L. As well as these: use Mojo::Base 'EP::Visualizer::base'; use Mojo::Util qw(url_unescape); use Mojo::URL; -use Mojo::JSON; +use Mojo::JSON qw(decode_json); +use Data::Dumper; use Mojo::UserAgent; use Mojo::Template; @@ -64,8 +65,6 @@ use POSIX qw(strftime); has 'hostauth'; has view => 'embedded'; -has json => sub {Mojo::JSON->new}; - sub new { my $self = shift->SUPER::new(@_); if( defined($self->cfg->{hostauth}) ){ @@ -157,6 +156,7 @@ sub rpcService { Gend => int($end), Gstep =>int($step || 1), Gmaxrows => 4000, + DATAONLY => 1, nodeid=>$item->{node_id}.'//'.$view ); $self->app->log->debug($url->to_string); @@ -167,36 +167,42 @@ sub rpcService { if (my $res=$tx->success) { if ($res->headers->content_type =~ m'application/json'i){ # $self->app->log->debug($res->body); - my $ret = $self->json->decode($res->body); + my $ret = eval { decode_json($res->body) }; + if ($@){ + $self->log->error("Error JSON Decode:".$@); + push @$return, { + status => 'failed', + message => "JSON Decode Problem $url: $@" + }; + next; + } if ($ret->{success}){ push @$return, { status => 'ok', start => $ret->{data}{start}, step => $ret->{data}{step}, values => [ map { $dataExtract ? $dataExtract->(@$_) : $_->[0] } @{$ret->{data}{data}} ], - } - } - else { - push @$return, { - status => 'failed', - message => "Torrus Problem $url: $ret->{error}" }; + next; } - } - else { + $self->log->error("Torrus Problem $url: $ret->{error}"); push @$return, { - status => 'failed', - message => "Faild Fetch $url: Data Type ".$res->headers->content_type, + status => 'failed', + message => "Torrus Problem $url: $ret->{error}" }; + next; } - } - else { - my $error = $tx->error; push @$return, { - status => 'faild', - message => "Faild Fetch $url: $error->{message}", + status => 'failed', + message => "Faild Fetch $url: Data Type ".$res->headers->content_type, }; + next; } + my $error = $tx->error; + push @$return, { + status => 'failed', + message => "Faild Fetch $url: $error->{message}", + }; } return $return } diff --git a/software/backend/lib/EP/Visualizer/TorrusBrowserChart.pm b/software/backend/lib/EP/Visualizer/TorrusBrowserChart.pm index f945316..b35163b 100644 --- a/software/backend/lib/EP/Visualizer/TorrusBrowserChart.pm +++ b/software/backend/lib/EP/Visualizer/TorrusBrowserChart.pm @@ -52,7 +52,7 @@ all the methods from L. As well as these: use Mojo::Base 'EP::Visualizer::TorrusChart'; use EP::Exception qw(mkerror); - +use Mojo::JSON qw(decode_json); =head2 matchRecord(type,args) @@ -125,7 +125,7 @@ sub getChartData { if (my $res=$tx->success) { if ($res->headers->content_type =~ m'application/json'i){ # $self->app->log->debug($res->body); - my $ret = eval { $self->json->decode($res->body) }; + my $ret = eval { decode_json($res->body) }; if ($@){ return { status => 'failed', diff --git a/software/backend/lib/EP/Visualizer/TorrusChart.pm b/software/backend/lib/EP/Visualizer/TorrusChart.pm index 0f8ba7a..da60d94 100644 --- a/software/backend/lib/EP/Visualizer/TorrusChart.pm +++ b/software/backend/lib/EP/Visualizer/TorrusChart.pm @@ -80,7 +80,7 @@ all the methods from L. As well as these: use Mojo::Base 'EP::Visualizer::base'; use Mojo::Util qw(url_unescape); use Mojo::URL; -use Mojo::JSON; +use Mojo::JSON qw(decode_json); use Mojo::UserAgent; use Mojo::Template; @@ -90,7 +90,7 @@ use POSIX qw(strftime); has 'hostauth'; has view => 'embedded'; -has json => sub {Mojo::JSON->new}; + has 'printtemplate'; has 'mode' => 'traffic'; has root => sub {'torrusChart_'.shift->instance}; @@ -219,7 +219,11 @@ sub getLeaves { my $tx = Mojo::UserAgent->new->get($url); if (my $res=$tx->success) { if ($res->headers->content_type =~ m'application/json'i){ - my $ret = $self->json->decode($res->body); + my $ret = eval { decode_json($res->body) }; + if ($@){ + $log->error("Running $rpcCall on ".join(', ',map{"$_: $callParams->{$_}"} keys %$callParams).$@); + return {}; + } if ($ret->{success}){ return $ret->{data}; } else { diff --git a/software/backend/lib/EP/Visualizer/TorrusData.pm b/software/backend/lib/EP/Visualizer/TorrusData.pm index 8bce49d..3254584 100644 --- a/software/backend/lib/EP/Visualizer/TorrusData.pm +++ b/software/backend/lib/EP/Visualizer/TorrusData.pm @@ -51,7 +51,7 @@ all the methods from L. As well as these: use Mojo::Base 'EP::Visualizer::base'; use Mojo::Util qw(url_unescape); use Mojo::URL; -use Mojo::JSON; +use Mojo::JSON qw(decode_json); use Mojo::UserAgent; use Mojo::Template; @@ -68,7 +68,7 @@ has 'hostauth'; has root => sub {'torrusCSV_'.shift->instance }; has view => 'embedded'; -has json => sub { Mojo::JSON->new}; + sub new { my $self = shift->SUPER::new(@_); @@ -279,7 +279,14 @@ sub getData { my $data; if (my $res=$tx->success) { if ($res->headers->content_type =~ m'application/json'i){ - my $ret = $self->json->decode($res->body); + my $ret = eval { decode_json($res->body) }; + if ($@){ + $self->app->log->error("Fetching ".$url->to_string.": ".$@); + return { + status => 0, + error => $@ + }; + } if ($ret->{success}){ my $key = (keys %{$ret->{data}})[0]; $data{$subNode} = rrd2float($ret->{data}{$key}); diff --git a/software/backend/lib/EP/Visualizer/TorrusIframe.pm b/software/backend/lib/EP/Visualizer/TorrusIframe.pm index 6ba4850..3141a82 100644 --- a/software/backend/lib/EP/Visualizer/TorrusIframe.pm +++ b/software/backend/lib/EP/Visualizer/TorrusIframe.pm @@ -38,7 +38,6 @@ use EP::Exception qw(mkerror); has 'hostauth'; #has 'view' => 'expanded-dir-html'; has view => 'iframe-rrd'; -has json => sub {Mojo::JSON->new}; sub new { @@ -109,7 +108,11 @@ sub getLeaves { my $tx = Mojo::UserAgent->new->get($url); if (my $res=$tx->success) { if ($res->headers->content_type =~ m'application/json'i){ - my $ret = $self->json->decode($res->body); + my $ret = eval { decode_json($res->body) }; + if ($@){ + $log->error("JSON decode Problem:".$@); + return {}; + } if ($ret->{success}){ return $ret->{data}; } else { diff --git a/software/backend/lib/EP/Visualizer/base.pm b/software/backend/lib/EP/Visualizer/base.pm index d0fd71c..924dade 100644 --- a/software/backend/lib/EP/Visualizer/base.pm +++ b/software/backend/lib/EP/Visualizer/base.pm @@ -37,6 +37,16 @@ a pointer to the application object has 'app'; +=head2 log + +log object + +=cut + +has log => sub { + shift->app->log; +}; + =head2 instance the name of the instance @@ -71,7 +81,7 @@ used on extopus side. It returns either undef (no match) or an array of maps: sub matchRecord { my $self= shift; - my $rec = shift; + my $rec = shift; return; } @@ -83,7 +93,7 @@ Can the Visualizer deal with multiple records of the given type? sub matchMultiRecord { my $self= shift; - my $rec = shift; + my $rec = shift; return; } @@ -115,7 +125,7 @@ rpcService wants to provide. The config option is called caption_live. sub caption_live { my $self = shift; my $rec = shift; - + return $self->caption($rec) unless $self->cfg->{caption_live}; my $conf = shift; @@ -137,7 +147,7 @@ sub rpcService { ## no critic (RequireArgUnpacking) my $self = shift; my $controller = shift; my @args = @_; - die "sorry, no rpc service support"; + die "sorry, no rpc service support"; } =head2 calcHash(ref) @@ -148,7 +158,7 @@ Returns a hash for authenticating access to the ref sub calcHash { ## no critic (RequireArgUnpacking) my $self = shift; - # $self->log->debug('HASH '.join(',',@_)); + # $self->log->debug('HASH '.join(',',@_)); my $hash = hmac_sha1_sum(join('::',@_),$self->app->secrets->[0]); return $hash; } diff --git a/software/frontend/source/class/ep/visualizer/chart/BrowserChart.js b/software/frontend/source/class/ep/visualizer/chart/BrowserChart.js index c3841b3..f4dab49 100644 --- a/software/frontend/source/class/ep/visualizer/chart/BrowserChart.js +++ b/software/frontend/source/class/ep/visualizer/chart/BrowserChart.js @@ -68,7 +68,7 @@ qx.Class.define("ep.visualizer.chart.BrowserChart", { d3Obj.addListener('resize',this.setSize,this); - var timer = this.__timer = new qx.event.Timer(60 * 1000); + var timer = this.__timer = new qx.event.Timer(5 * 1000); d3Obj.addListener('disappear', function() { timer.stop() @@ -197,6 +197,7 @@ qx.Class.define("ep.visualizer.chart.BrowserChart", { __clipPath: null, __chartWidth: null, __fetchWait: null, + __fetchAgain: null, __legendContainer: null, __d3Obj: null, __data: null, @@ -361,7 +362,7 @@ qx.Class.define("ep.visualizer.chart.BrowserChart", { .attr("fill",chartDef.color); break; default: - this.debug("invalid cmd"); + this.debug("unsupported cmd:" + chartDef.cmd); break; } return this.__dataNode[id]; @@ -469,8 +470,8 @@ qx.Class.define("ep.visualizer.chart.BrowserChart", { dates[1] = new Date(); dates[0] = new Date(dates[1].getTime() - interval); this.getXScale().domain(dates); + this.redraw(); } - this.redraw(); }, yScaleRedraw: function(){ @@ -502,7 +503,7 @@ qx.Class.define("ep.visualizer.chart.BrowserChart", { var dates = this.getXScale().domain(); var start = Math.round(dates[0].getTime()/1000); var end = Math.round(dates[1].getTime()/1000); - var dataStep = (end-start )/ this.__chartWidth; + var dataStep = Math.round((end-start)/ this.__chartWidth); var extra = Math.round(end - start ); var d3 = this.__d3; @@ -520,11 +521,16 @@ qx.Class.define("ep.visualizer.chart.BrowserChart", { return; } + var existingData = this.dataSlicer(start,end,dataStep); + if (existingData.missingStart == existingData.missingEnd){ + return; + } + + var needChartDef = this.getChartDef().length == 0; + var rpc = ep.data.Server.getInstance(); var that = this; this.__fetchWait = 1; - var existingData = this.dataSlicer(start,end,dataStep); - var needChartDef = this.getChartDef().length == 0; rpc.callAsyncSmart(function(ret){ var d3Data = that.__data = that.d3DataTransformer(ret,dataStep); @@ -540,7 +546,6 @@ qx.Class.define("ep.visualizer.chart.BrowserChart", { } that.yScaleRedraw(); - that.__fetchWait = 0; // if we skipped one, lets redraw again just to be sure we got it all if (this.__fetchAgain == 1){ @@ -565,38 +570,38 @@ qx.Class.define("ep.visualizer.chart.BrowserChart", { var missingEnd = end; var prepend = []; var append = []; - var keepData = oldData && Math.round(oldData.dataStep) == Math.round(dataStep); + var keepData = oldData != null && typeof(oldData) == 'object' && Math.round(oldData.dataStep) == Math.round(dataStep); if (!keepData) { return { missingStart: missingStart, missingEnd: missingEnd}; } - var prependMode = keepData && oldData.data[0][0].date.getTime()/1000 <= start; - var appendMode = keepData && oldData.data[0][oldData.data[0].length-1].date.getTime()/1000 >= end; + var oldStart = oldData.data[0][0].date.getTime()/1000; + var oldEnd = oldData.data[0][oldData.data[0].length-1].date.getTime()/1000; + // prepend the existing data to the new data + var prependMode = oldStart <= start && oldEnd >= start; + var appendMode = oldEnd >= end && oldStart <= end; for (var i=0;i= start ) { prepend[i].push(item); - missingStart = date; + missingStart = date+dataStep; } if (appendMode && date <= end ){ append[i].push(item); - if ( missingEnd > date) { - missingEnd = date; + if ( date < missingEnd ) { + missingEnd = date-dataStep; } } } } /* lets make sure don't trip over ourselves */ if (missingStart > missingEnd){ - if (appendMode){ - missingEnd = missingStart + dataStep; - } - else { - missingStart = missingEnd - dataStep; - } + this.debug("missingStart:"+missingStart+" missingEnd:"+missingEnd); + missingEnd = missingStart; } return { @@ -609,33 +614,40 @@ qx.Class.define("ep.visualizer.chart.BrowserChart", { d3DataTransformer: function(data,dataStep){ var d3 = this.__d3; - var minStep = data[0].step; - var minStart = data[0].start; + if (data == null || typeof(data) != 'object' || data[0] == null){ + return null; + } + var minStep = 24*3600; + var minStart = (new Date).getTime()/1000; data.forEach(function(d){ if (d.status != 'ok') return; - if (minStep > d.step){ + if (minStep == null || minStep > d.step){ minStep = d.step; minStart = d.start; } }); + var d3Data = []; for (var i=0; i 0 ){ + if (stack && i > 0 && d3Data[i-1][ii] != null && typeof(d3Data[i-1][ii]) == 'object'){ y0 = d3Data[i-1][ii].y; } - var yval = data[i].values[Math.round(ii*st)]; + var yval = parseFloat(data[i].values[Math.round(ii*st)]); d3Data[i][ii] = { - y: yval+y0, + y: (isNaN(yval) ? 0 : yval) +y0, y0: y0, - d: yval !== null, + d: !isNaN(yval), date: new Date((minStart+ii*minStep)*1000) } }