diff --git a/infer/man/man1/infer-full.txt b/infer/man/man1/infer-full.txt index 114da52d614..7cc7d23bece 100644 --- a/infer/man/man1/infer-full.txt +++ b/infer/man/man1/infer-full.txt @@ -2140,6 +2140,17 @@ OPTIONS (Conversely: --no-starvation-only) See also infer-analyze(1). + --stats-dir-current path + The infer-out/stats from the current run. Together with + --stats-dir-previous, make infer reportdiff compute the difference + between two stats directories and output the results in + infer-out/differential/stats_*.json files. + See also infer-reportdiff(1). + + --stats-dir-previous path + The infer-out/stats from a previous run. See --stats-dir-current. + See also infer-reportdiff(1). + --store-analysis-schedule Activates: Store the analysis schedule for later replay, honoring --replay-analysis-schedule-file if present. This can be useful to @@ -3266,6 +3277,12 @@ INTERNAL OPTIONS Activates: Run whole-program starvation analysis (Conversely: --no-starvation-whole-program) + --stats-dir-current-reset + Cancel the effect of --stats-dir-current. + + --stats-dir-previous-reset + Cancel the effect of --stats-dir-previous. + --no-subtype-multirange Deactivates: Use the multirange subtyping domain. Used in the Java frontend and in biabduction. (Conversely: --subtype-multirange) diff --git a/infer/man/man1/infer-reportdiff.txt b/infer/man/man1/infer-reportdiff.txt index 2f1c44d7b6c..ddb59b06075 100644 --- a/infer/man/man1/infer-reportdiff.txt +++ b/infer/man/man1/infer-reportdiff.txt @@ -85,6 +85,15 @@ OPTIONS computing differential reports (Conversely: --skip-duplicated-types) + --stats-dir-current path + The infer-out/stats from the current run. Together with + --stats-dir-previous, make infer reportdiff compute the difference + between two stats directories and output the results in + infer-out/differential/stats_*.json files. + + --stats-dir-previous path + The infer-out/stats from a previous run. See --stats-dir-current. + ENVIRONMENT INFER_ARGS, INFERCONFIG, INFER_STRICT_MODE See the ENVIRONMENT section in the manual of infer(1). diff --git a/infer/man/man1/infer.txt b/infer/man/man1/infer.txt index 659548c1dcc..3bbbf97d535 100644 --- a/infer/man/man1/infer.txt +++ b/infer/man/man1/infer.txt @@ -2140,6 +2140,17 @@ OPTIONS (Conversely: --no-starvation-only) See also infer-analyze(1). + --stats-dir-current path + The infer-out/stats from the current run. Together with + --stats-dir-previous, make infer reportdiff compute the difference + between two stats directories and output the results in + infer-out/differential/stats_*.json files. + See also infer-reportdiff(1). + + --stats-dir-previous path + The infer-out/stats from a previous run. See --stats-dir-current. + See also infer-reportdiff(1). + --store-analysis-schedule Activates: Store the analysis schedule for later replay, honoring --replay-analysis-schedule-file if present. This can be useful to diff --git a/infer/src/base/Config.ml b/infer/src/base/Config.ml index f78f182ac12..a20bfd28692 100644 --- a/infer/src/base/Config.ml +++ b/infer/src/base/Config.ml @@ -3403,6 +3403,20 @@ and starvation_whole_program = "Run whole-program starvation analysis" +and stats_dir_current = + CLOpt.mk_path_opt ~long:"stats-dir-current" + ~in_help:InferCommand.[(ReportDiff, manual_generic)] + "The infer-out/stats from the current run. Together with $(b,--stats-dir-previous), make \ + $(i,infer reportdiff) compute the difference between two stats directories and output the \ + results in infer-out/differential/stats_*.json files." + + +and stats_dir_previous = + CLOpt.mk_path_opt ~long:"stats-dir-previous" + ~in_help:InferCommand.[(ReportDiff, manual_generic)] + "The infer-out/stats from a previous run. See $(b,--stats-dir-current)." + + and store_analysis_schedule = CLOpt.mk_bool ~long:"store-analysis-schedule" ~in_help:InferCommand.[(Analyze, manual_scheduler)] @@ -4673,6 +4687,10 @@ and starvation_strict_mode = !starvation_strict_mode and starvation_whole_program = !starvation_whole_program +and stats_dir_current = !stats_dir_current + +and stats_dir_previous = !stats_dir_previous + and store_analysis_schedule = !store_analysis_schedule and subtype_multirange = !subtype_multirange diff --git a/infer/src/base/Config.mli b/infer/src/base/Config.mli index fc71e39abdf..d7339f5b361 100644 --- a/infer/src/base/Config.mli +++ b/infer/src/base/Config.mli @@ -846,6 +846,10 @@ val starvation_strict_mode : bool val starvation_whole_program : bool +val stats_dir_current : string option + +val stats_dir_previous : string option + val store_analysis_schedule : bool val subtype_multirange : bool diff --git a/infer/src/base/ScubaLogging.ml b/infer/src/base/ScubaLogging.ml index dc27b78381a..5d63008116d 100644 --- a/infer/src/base/ScubaLogging.ml +++ b/infer/src/base/ScubaLogging.ml @@ -75,7 +75,7 @@ let log_to_debug samples = Utils.with_file_out ~append:true log_file ~f:(fun out -> List.iter samples ~f:(fun sample -> Yojson.to_channel out (Scuba.sample_to_json sample) ; - Printf.fprintf out ",\n" ) ) + Printf.fprintf out "\n" ) ) (** Consider buffering or batching if proves to be a problem *) diff --git a/infer/src/integration/InferCommandImplementation.ml b/infer/src/integration/InferCommandImplementation.ml index 03ddb03eb80..0d792a33325 100644 --- a/infer/src/integration/InferCommandImplementation.ml +++ b/infer/src/integration/InferCommandImplementation.ml @@ -301,23 +301,27 @@ let report () = let report_diff () = - (* at least one report must be passed in input to compute differential *) + (* at least one pair of reports must be passed as input to compute a differential *) + let open Config in match - Config. - ( report_current - , report_previous - , costs_current - , costs_previous - , config_impact_current - , config_impact_previous ) + ( Option.both report_current report_previous + , Option.both costs_current costs_previous + , Option.both config_impact_current config_impact_previous + , Option.both stats_dir_current stats_dir_previous ) with - | None, None, None, None, None, None -> + | None, None, None, None -> L.die UserError - "Expected at least one argument among '--report-current', '--report-previous', \ - '--costs-current', '--costs-previous', '--config-impact-current', and \ - '--config-impact-previous'\n" + "Expected at least one pair of arguments among '--report-current'/'--report-previous', \ + '--costs-current'/'--costs-previous', \ + '--config-impact-current'/'--config-impact-previous', or \ + '--stats-dir-current'/'--stats-dir-previous'" | _ -> - ReportDiff.reportdiff ~current_report:Config.report_current - ~previous_report:Config.report_previous ~current_costs:Config.costs_current - ~previous_costs:Config.costs_previous ~current_config_impact:Config.config_impact_current - ~previous_config_impact:Config.config_impact_previous + if + (is_some @@ Option.both report_current report_previous) + || (is_some @@ Option.both costs_current costs_previous) + || (is_some @@ Option.both config_impact_current config_impact_previous) + then + ReportDiff.reportdiff ~report_current ~report_previous ~costs_current ~costs_previous + ~config_impact_current ~config_impact_previous ; + Option.both stats_dir_previous stats_dir_current + |> Option.iter ~f:(fun (previous, current) -> StatsDiff.diff ~previous ~current) diff --git a/infer/src/integration/ReportDiff.ml b/infer/src/integration/ReportDiff.ml index ad8f7371588..d18e31cb535 100644 --- a/infer/src/integration/ReportDiff.ml +++ b/infer/src/integration/ReportDiff.ml @@ -4,12 +4,13 @@ * This source code is licensed under the MIT license found in the * LICENSE file in the root directory of this source tree. *) + open! IStd -let reportdiff ~current_report:current_report_fname ~previous_report:previous_report_fname - ~current_costs:current_costs_fname ~previous_costs:previous_costs_fname - ~current_config_impact:current_config_impact_fname - ~previous_config_impact:previous_config_impact_fname = +let reportdiff ~report_current:current_report_fname ~report_previous:previous_report_fname + ~costs_current:current_costs_fname ~costs_previous:previous_costs_fname + ~config_impact_current:current_config_impact_fname + ~config_impact_previous:previous_config_impact_fname = let load_aux ~f filename_opt = Option.value_map ~f:(fun filename -> Atdgen_runtime.Util.Json.from_file f filename) diff --git a/infer/src/integration/ReportDiff.mli b/infer/src/integration/ReportDiff.mli index a1f764eb801..45c05a3b7de 100644 --- a/infer/src/integration/ReportDiff.mli +++ b/infer/src/integration/ReportDiff.mli @@ -8,10 +8,10 @@ open! IStd val reportdiff : - current_report:string option - -> previous_report:string option - -> current_costs:string option - -> previous_costs:string option - -> current_config_impact:string option - -> previous_config_impact:string option + report_current:string option + -> report_previous:string option + -> costs_current:string option + -> costs_previous:string option + -> config_impact_current:string option + -> config_impact_previous:string option -> unit diff --git a/infer/src/integration/StatsDiff.ml b/infer/src/integration/StatsDiff.ml new file mode 100644 index 00000000000..b12e5b7adf6 --- /dev/null +++ b/infer/src/integration/StatsDiff.ml @@ -0,0 +1,148 @@ +(* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + *) + +open! IStd +module L = Logging + +(** + + entries are of the form + + + { + "int": { + "is_main_process": 1, + "pid": 1234, + "time": 420844200, + "value": 123456 + }, + "normal": { + "command": "run", + "event": "time.dbwriter.useful_time_user", + "hostname": "darkstar", + "infer_commit": "deadbeef" + }, + "tags": {} + } + *) + +let error ~expected json = + L.die InternalError "when parsing json: expected %s but got '%a'" expected Yojson.Safe.pp json + + +let get_field field_name json = + match json with + | `Assoc assoc -> + List.Assoc.find ~equal:String.equal assoc field_name + | _ -> + error ~expected:"record" json + + +let get_field_exn field_name json = + match get_field field_name json with + | Some json' -> + json' + | None -> + error ~expected:(Printf.sprintf "field '%s'" field_name) json + + +let get_string_exn json = match json with `String s -> s | json -> error ~expected:"string" json + +let get_event_exn json = json |> get_field_exn "normal" |> get_field_exn "event" |> get_string_exn + +let get_value_exn json = + (* some values are strings ("normal"), some are ints, none are neither *) + match json |> get_field_exn "normal" |> get_field "message" with + | Some value -> + value + | None -> + json |> get_field_exn "int" |> get_field_exn "value" + + +let collate_stats_in_dir stats_dir = + Iter.fold + (fun json_rows dir_entry -> + if Filename.check_suffix dir_entry ".jsonl" then + Iter.fold + (fun json_rows line -> Yojson.Safe.from_string line :: json_rows) + json_rows + (Iter.from_labelled_iter @@ In_channel.iter_lines @@ In_channel.create dir_entry) + else json_rows ) + [] + (Iter.from_labelled_iter @@ Utils.iter_dir stats_dir) + + +let diff_values entry1 entry2 = + let v1 = get_value_exn entry1 in + let v2 = get_value_exn entry2 in + if Yojson.Safe.equal v1 v2 then `Same v1 else `Diff (v1, v2) + + +let compute_diff ~before ~after = + let sort_entries entries = + List.sort + ~compare:(fun json1 json2 -> String.compare (get_event_exn json1) (get_event_exn json2)) + entries + in + let before = sort_entries before in + let after = sort_entries after in + let rec diff_aux ~extra_before ~extra_after ~unchanged ~diff before after = + match (before, after) with + | [], _ -> + (extra_before, after @ extra_after, unchanged, diff) + | _, [] -> + (before @ extra_before, extra_after, unchanged, diff) + | entry1 :: before', entry2 :: after' -> + let event1 = get_event_exn entry1 in + let event2 = get_event_exn entry2 in + let cmp = String.compare event1 event2 in + if Int.equal cmp 0 then + (* [event1 = event2] *) + match diff_values entry1 entry2 with + | `Same v -> + diff_aux ~extra_before ~extra_after ~unchanged:((event1, v) :: unchanged) ~diff + before' after' + | `Diff d -> + diff_aux ~extra_before ~extra_after ~unchanged ~diff:((event1, d) :: diff) before' + after' + else + let extra_before, extra_after, before'', after'' = + if cmp < 0 then + ((* [event1 < event2] *) + entry1 :: extra_before, extra_after, before', after) + else ((* [event1 > event2] *) extra_before, entry2 :: extra_after, before, after') + in + diff_aux ~extra_before ~extra_after ~unchanged ~diff before'' after'' + in + let extra_before, extra_after, unchanged, diff = + diff_aux ~extra_before:[] ~extra_after:[] ~unchanged:[] ~diff:[] before after + in + ( `List extra_before + , `List extra_after + , `List + (List.map unchanged ~f:(fun (event, value) -> + `Assoc [("event", `String event); ("value", value)] ) ) + , `List + (List.map diff ~f:(fun (event, (before, after)) -> + `Assoc [("event", `String event); ("value_before", before); ("value_after", after)] ) ) + ) + + +let output_diff (extra_before, extra_after, unchanged, diff) = + let out_dir = ResultsDir.get_path Differential in + Unix.mkdir_p out_dir ; + Yojson.Safe.to_file (out_dir ^/ "stats_previous_only.json") extra_before ; + Yojson.Safe.to_file (out_dir ^/ "stats_current_only.json") extra_after ; + Yojson.Safe.to_file (out_dir ^/ "stats_unchanged.json") unchanged ; + Yojson.Safe.to_file (out_dir ^/ "stats_diff.json") diff ; + () + + +let diff ~previous:stats_dir_previous ~current:stats_dir_current = + let stats_previous = collate_stats_in_dir stats_dir_previous in + let stats_current = collate_stats_in_dir stats_dir_current in + compute_diff ~before:stats_previous ~after:stats_current |> output_diff diff --git a/infer/src/integration/StatsDiff.mli b/infer/src/integration/StatsDiff.mli new file mode 100644 index 00000000000..e694a1e43a0 --- /dev/null +++ b/infer/src/integration/StatsDiff.mli @@ -0,0 +1,10 @@ +(* + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + *) + +open! IStd + +val diff : previous:string -> current:string -> unit