dc add to late packet (#1001)

* send dc balance to charge for late packets

* better failure formatting

* fix test for bigger fcnt search space

* send the verified fcnt

* test for very late packets

* update tests and add env vars for charging late packets

* remove offer tracking test, dup of multi buy test

* update eunit for expecting records

* require dc map

* ensure dc used is 0 if not charging for late packets

* add callout of settings at beginning of test

- to ensure that settings is present despite how test.config may change.
- change fcnt of second replay to make debugging easier

* show payload and size for late packet when charged

* better handle default env bool case

.env-template

@@ -68,3 +68,6 @@ ROUTER_DEVADDR_PREFIX=72
 
 # Charge an Organization for join requests (default: true)
 ROUTER_CHARGE_JOINS=true
+
+# Charge an org for a packet received late or replayed (default: false).
+ROUTER_CHARGE_LATE_PACKETS=false

config/sys.config.src

@@ -69,7 +69,8 @@
         {metrics_port, "${ROUTER_METRICS_PORT}"},
         {denylist_keys, ["1SbEYKju337P6aYsRd9DT2k4qgK5ZK62kXbSvnJgqeaxK3hqQrYURZjL"]},
         {denylist_url, "https://api.github.com/repos/helium/denylist/releases/latest"},
-        {charge_joins, "${ROUTER_CHARGE_JOINS}"}
+        {charge_joins, "${ROUTER_CHARGE_JOINS}"},
+        {charge_late_packets, "${ROUTER_CHARGE_LATE_PACKETS}"}
     ]},
     {grpcbox, [
         {client, #{

config/test.config

@@ -8,6 +8,7 @@
         {router_http_channel_url_check, false},
         {router_xor_filter_worker, false},
         {charge_when_no_offer, true},
+        {charge_late_packets, true},
         {ics, #{
             transport => http,
             host => "localhost",

src/apis/router_console_api.erl

@@ -324,7 +324,8 @@ event(Device, Map) ->
                         #{
                             fcnt => maps:get(fcnt, Map),
                             hotspot => maps:get(hotspot, Map),
-                            hold_time => maps:get(hold_time, Map)
+                            hold_time => maps:get(hold_time, Map),
+                            dc => maps:get(dc, Map)
                         };
                     {uplink_dropped, _SC} ->
                         #{

src/apis/router_console_dc_tracker.erl

@@ -16,6 +16,7 @@
     has_enough_dc/2,
     charge/2,
     current_balance/1,
+    maybe_charge_late/2,
     %%
     add_unfunded/1,
     remove_unfunded/1,
@@ -177,6 +178,30 @@ current_balance(OrgID) ->
             {Balance, Nonce}
     end.
 
+-spec maybe_charge_late(
+    Device :: router_device:device(),
+    Packet :: blockchain_helium_packet_v1:packet()
+) ->
+    {ok, Balance :: non_neg_integer(), Nonce :: non_neg_integer(), Used :: non_neg_integer()}
+    | {error, any()}.
+maybe_charge_late(Device, Packet) ->
+    OrgID = router_device:organization_id(Device),
+    case router_utils:get_env_bool(charge_late_packets, false) of
+        false ->
+            {Balance, Nonce} = current_balance(OrgID),
+            {ok, Balance, Nonce, 0};
+        true ->
+            Payload = blockchain_helium_packet_v1:payload(Packet),
+            PayloadSize = erlang:byte_size(Payload),
+            case charge(Device, PayloadSize) of
+                {error, _} = E ->
+                    E;
+                {ok, Balance, Nonce} ->
+                    Used = router_blockchain:calculate_dc_amount(PayloadSize),
+                    {ok, Balance, Nonce, Used}
+            end
+    end.
+
 %% ------------------------------------------------------------------
 %% gen_server Function Definitions
 %% ------------------------------------------------------------------

src/device/router_device.erl

@@ -50,7 +50,8 @@
     credentials_to_evict/1,
     limit_credentials/1,
     devaddr_int_nwk_key/1,
-    make_skf_removes/1, make_skf_removes/2, make_skf_removes/3
+    make_skf_removes/1, make_skf_removes/2, make_skf_removes/3,
+    organization_id/1
 ]).
 
 %% ------------------------------------------------------------------
@@ -566,6 +567,10 @@ make_skf_removes(NwkKeys, DevAddrs, MultiBuy) ->
 
     lists:usort([{remove, DevAddrToInt(D), NSK, MultiBuy} || {NSK, _} <- NwkKeys, D <- DevAddrs]).
 
+-spec organization_id(device()) -> undefined | binary().
+organization_id(#device_v7{metadata = Metadata}) ->
+    maps:get(organization_id, Metadata, undefined).
+
 %% ------------------------------------------------------------------
 %% RocksDB Device Functions
 %% ------------------------------------------------------------------

src/device/router_device_routing.erl

@@ -39,6 +39,12 @@
     force_evict_packet_hash/1
 ]).
 
+-export([
+    payload_b0/2,
+    payload_mic/1,
+    payload_fcnt_low/1
+]).
+
 %% biggest unsigned number in 23 bits
 -define(BITS_23, 8388607).
 -define(MODULO_16_BITS, 16#10000).
@@ -221,24 +227,15 @@ handle_free_packet(SCPacket, PacketTime, Pid) when is_pid(Pid) ->
                 ok = lager:md([{device_id, router_device:id(Device)}]),
                 case validate_payload_for_device(Device, Payload, PHash, PubKeyBin, PacketFCnt) of
                     ok ->
-                        Offer = blockchain_state_channel_offer_v1:from_packet(
-                            Packet, PubKeyBin, Region
-                        ),
-                        case offer_check(Offer) of
-                            ok ->
-                                erlang:spawn(router_blockchain, track_offer, [Offer, Pid]),
-                                {ok, Device};
-                            {error, _} ->
-                                lager:debug("packet accepted even if on denylist"),
-                                {ok, Device}
-                        end;
+                        {ok, Device};
                     {error, ?LATE_PACKET} = E2 ->
                         ok = router_utils:event_uplink_dropped_late_packet(
                             PacketTime,
                             HoldTime,
                             PacketFCnt,
                             Device,
-                            PubKeyBin
+                            PubKeyBin,
+                            Packet
                         ),
                         E2;
                     {error, _} = E3 ->
@@ -1187,7 +1184,7 @@ get_device_by_mic(_MIC, _Payload, []) ->
     undefined;
 get_device_by_mic(ExpectedMIC, Payload, [Device | Devices]) ->
     ExpectedFCnt = expected_fcnt(Device, Payload),
-    B0 = b0_from_payload(Payload, ExpectedFCnt),
+    B0 = payload_b0(Payload, ExpectedFCnt),
     DeviceID = router_device:id(Device),
     case router_device:nwk_s_key(Device) of
         undefined ->
@@ -1235,10 +1232,31 @@ find_right_key(B0, MIC, Payload, Device, [{undefined, _} | Keys]) ->
     find_right_key(B0, MIC, Payload, Device, Keys);
 find_right_key(B0, MIC, Payload, Device, [{NwkSKey, _} | Keys]) ->
     case key_matches_mic(NwkSKey, B0, MIC) of
-        true -> {Device, NwkSKey};
-        false -> find_right_key(B0, MIC, Payload, Device, Keys)
+        true ->
+            {Device, NwkSKey};
+        false ->
+            case key_matches_any_fcnt(NwkSKey, MIC, Payload) of
+                false -> find_right_key(B0, MIC, Payload, Device, Keys);
+                true -> {Device, NwkSKey}
+            end
     end.
 
+-spec key_matches_any_fcnt(binary(), binary(), binary()) -> boolean().
+key_matches_any_fcnt(NwkSKey, ExpectedMIC, Payload) ->
+    FCntLow = payload_fcnt_low(Payload),
+    lists:any(
+        fun(HighBits) ->
+            FCnt = binary:decode_unsigned(
+                <<FCntLow:16/integer-unsigned-little, HighBits:16/integer-unsigned-little>>,
+                little
+            ),
+            B0 = payload_b0(Payload, FCnt),
+            ComputedMIC = crypto:macN(cmac, aes_128_cbc, NwkSKey, B0, 4),
+            ComputedMIC =:= ExpectedMIC
+        end,
+        lists:seq(2#000, 2#111)
+    ).
+
 -spec key_matches_mic(binary(), binary(), binary()) -> boolean().
 key_matches_mic(Key, B0, ExpectedMIC) ->
     ComputedMIC = crypto:macN(cmac, aes_128_cbc, Key, B0, 4),
@@ -1316,8 +1334,8 @@ expected_fcnt(Device, Payload) ->
             binary:decode_unsigned(<<NewHigh:16, PayloadFCntLow:16>>)
     end.
 
--spec b0_from_payload(binary(), non_neg_integer()) -> binary().
-b0_from_payload(Payload, ExpectedFCnt) ->
+-spec payload_b0(binary(), non_neg_integer()) -> binary().
+payload_b0(Payload, ExpectedFCnt) ->
     <<MType:3, _:5, DevAddr:4/binary, _:4, FOptsLen:4, _:16, _FOpts:FOptsLen/binary, _/binary>> =
         Payload,
     Msg = binary:part(Payload, {0, erlang:byte_size(Payload) - 4}),

src/device/router_device_worker.erl

@@ -245,7 +245,8 @@ handle_call(
                                 HoldTime,
                                 PacketFCnt,
                                 Device,
-                                PubKeyBin
+                                PubKeyBin,
+                                Packet
                             ),
                             {reply, false, State};
                         true ->
@@ -836,13 +837,14 @@ handle_cast(
         {error, Reason} ->
             lager:debug("packet not validated: ~p", [Reason]),
             case Reason of
-                late_packet ->
+                {late_packet, VerifiedFCnt} ->
                     ok = router_utils:event_uplink_dropped_late_packet(
                         PacketTime,
                         HoldTime,
-                        PacketFCnt,
+                        VerifiedFCnt,
                         Device1,
-                        PubKeyBin
+                        PubKeyBin,
+                        Packet0
                     );
                 _ ->
                     ok = router_utils:event_uplink_dropped_invalid_packet(
@@ -1545,9 +1547,19 @@ validate_frame(
     FrameCache,
     OfferCache
 ) ->
-    <<MType:3, _MHDRRFU:3, _Major:2, _DevAddr:4/binary, _ADR:1, _ADRACKReq:1, _ACK:1, _RFU:1,
-        _FOptsLen:4, _FCnt:16, _FOpts:_FOptsLen/binary,
-        _PayloadAndMIC/binary>> = blockchain_helium_packet_v1:payload(Packet),
+    Payload =
+        <<MType:3, _MHDRRFU:3, _Major:2, _DevAddr:4/binary, _ADR:1, _ADRACKReq:1, _ACK:1, _RFU:1,
+            _FOptsLen:4, _FCnt:16, _FOpts:_FOptsLen/binary,
+            _PayloadAndMIC/binary>> = blockchain_helium_packet_v1:payload(Packet),
+
+    VerifiedFCnt = verified_fcnt_from_payload(
+        router_device_routing:payload_fcnt_low(Payload),
+        router_device:nwk_s_key(Device0),
+        router_device_routing:payload_mic(Payload),
+        Payload
+    ),
+    DeviceFCnt = router_device:fcnt(Device0),
+
     case MType of
         MType when MType == ?CONFIRMED_UP orelse MType == ?UNCONFIRMED_UP ->
             FrameAck = router_utils:mtype_to_ack(MType),
@@ -1579,7 +1591,7 @@ validate_frame(
                         PacketFCnt,
                         LastSeenFCnt
                     ]),
-                    {error, late_packet};
+                    {error, {late_packet, VerifiedFCnt}};
                 undefined when
                     FrameAck == 1 andalso PacketFCnt == DownlinkHandledAtFCnt andalso
                         Window < ?RX_MAX_WINDOW
@@ -1588,7 +1600,7 @@ validate_frame(
                         "we got a late confirmed up packet for ~p: DownlinkHandledAt: ~p within window ~p",
                         [PacketFCnt, DownlinkHandledAtFCnt, Window]
                     ),
-                    {error, late_packet};
+                    {error, {late_packet, VerifiedFCnt}};
                 undefined when
                     FrameAck == 1 andalso PacketFCnt == DownlinkHandledAtFCnt andalso
                         Window >= ?RX_MAX_WINDOW
@@ -1606,6 +1618,12 @@ validate_frame(
                         OfferCache,
                         true
                     );
+                undefined when VerifiedFCnt < DeviceFCnt ->
+                    lager:info(
+                        "we got a replay packet [verified: ~p] [device: ~p]",
+                        [VerifiedFCnt, DeviceFCnt]
+                    ),
+                    {error, {late_packet, VerifiedFCnt}};
                 undefined ->
                     lager:debug("we got a fresh packet [fcnt: ~p]", [PacketFCnt]),
                     validate_frame_(
@@ -2415,6 +2433,32 @@ maybe_will_downlink(Device, #frame{mtype = MType, adrackreq = ADRAckReqBit}) ->
     ADR = ADRAllowed andalso ADRAckReqBit == 1,
     DeviceQueue =/= [] orelse ACK == 1 orelse ADR orelse ChannelCorrection == false.
 
+-spec verified_fcnt_from_payload(non_neg_integer(), binary(), binary(), binary()) ->
+    non_neg_integer().
+verified_fcnt_from_payload(FCntLow, NwkSKey, ExpectedMIC, Payload) ->
+    find_first(
+        fun(HighBits) ->
+            FCnt = binary:decode_unsigned(
+                <<FCntLow:16/integer-unsigned-little, HighBits:16/integer-unsigned-little>>,
+                little
+            ),
+            B0 = router_device_routing:payload_b0(Payload, FCnt),
+            ComputedMIC = crypto:macN(cmac, aes_128_cbc, NwkSKey, B0, 4),
+            {ComputedMIC =:= ExpectedMIC, FCnt}
+        end,
+        lists:seq(2#000, 2#111)
+    ).
+
+-spec find_first(
+    FN :: fun((HighBit :: non_neg_integer()) -> {Verified :: boolean(), FCnt :: non_neg_integer()}),
+    HighBits :: list(non_neg_integer())
+) -> non_neg_integer().
+find_first(Fn, [FCntHigh | Rest]) ->
+    case Fn(FCntHigh) of
+        {true, Found} -> Found;
+        _ -> find_first(Fn, Rest)
+    end.
+
 -ifdef(TEST).
 -include_lib("eunit/include/eunit.hrl").
 

src/grpc/router_ics_skf_worker.erl

@@ -536,21 +536,29 @@ dedup_updates(Updates) ->
 -include_lib("eunit/include/eunit.hrl").
 
 dedup_updates_test() ->
+    MakeUpdate = fun(Action, Devaddr, SessionKey, MaxCopies) ->
+        #iot_config_route_skf_update_v1_pb{
+            action = Action,
+            devaddr = Devaddr,
+            session_key = SessionKey,
+            max_copies = MaxCopies
+        }
+    end,
     ?assertEqual(
-        [{add, 1, <<>>, 5}],
-        dedup_updates([{add, 1, <<>>, 5}]),
+        [MakeUpdate(add, 1, <<>>, 5)],
+        dedup_updates([MakeUpdate(add, 1, <<>>, 5)]),
         "adds are untouched"
     ),
 
     ?assertEqual(
-        [{add, 1, <<>>, 5}],
-        dedup_updates([{add, 1, <<>>, 5}, {remove, 1, <<>>, 9999}]),
+        [MakeUpdate(add, 1, <<>>, 5)],
+        dedup_updates([MakeUpdate(add, 1, <<>>, 5), MakeUpdate(remove, 1, <<>>, 9999)]),
         "removes that match an add are removed"
     ),
 
     ?assertEqual(
-        [{add, 1, <<>>, 5}],
-        dedup_updates([{remove, 1, <<>>, 9999}, {add, 1, <<>>, 5}]),
+        [MakeUpdate(add, 1, <<>>, 5)],
+        dedup_updates([MakeUpdate(remove, 1, <<>>, 9999), MakeUpdate(add, 1, <<>>, 5)]),
         "removes that match an add are removed regardless of order"
     ),