All: Fix left-shifts of signed vs unsigned constants

In order to silence -fsanitize=undefined all left shifts of constants
are now cast before shifting, in order to avoid undefined behavior,
if the target variable is unsigned.

Signed-off-by: Andreas Fuchs <andreas.fuchs@infineon.com>

include/tss2/tss2_tpm2_types.h

@@ -671,13 +671,13 @@ typedef TPM2_HANDLE TPM2_HC;
 #define TPM2_HR_HANDLE_MASK       ((TPM2_HC) 0x00FFFFFF) /* to mask off the HR */
 #define TPM2_HR_RANGE_MASK        ((TPM2_HC) 0xFF000000) /* to mask off the variable part */
 #define TPM2_HR_SHIFT             ((TPM2_HC) 24)
-#define TPM2_HR_PCR               ((TPM2_HC) (TPM2_HT_PCR << TPM2_HR_SHIFT))
-#define TPM2_HR_HMAC_SESSION      ((TPM2_HC) (TPM2_HT_HMAC_SESSION << TPM2_HR_SHIFT))
-#define TPM2_HR_POLICY_SESSION    ((TPM2_HC) (TPM2_HT_POLICY_SESSION << TPM2_HR_SHIFT))
-#define TPM2_HR_TRANSIENT         ((TPM2_HC) (TPM2_HT_TRANSIENT << TPM2_HR_SHIFT))
-#define TPM2_HR_PERSISTENT        ((TPM2_HC) (TPM2_HT_PERSISTENT << TPM2_HR_SHIFT))
-#define TPM2_HR_NV_INDEX          ((TPM2_HC) (TPM2_HT_NV_INDEX << TPM2_HR_SHIFT))
-#define TPM2_HR_PERMANENT         ((TPM2_HC) (TPM2_HT_PERMANENT << TPM2_HR_SHIFT))
+#define TPM2_HR_PCR               (((TPM2_HC) TPM2_HT_PCR) << TPM2_HR_SHIFT)
+#define TPM2_HR_HMAC_SESSION      (((TPM2_HC) TPM2_HT_HMAC_SESSION) << TPM2_HR_SHIFT)
+#define TPM2_HR_POLICY_SESSION    (((TPM2_HC) TPM2_HT_POLICY_SESSION) << TPM2_HR_SHIFT)
+#define TPM2_HR_TRANSIENT         (((TPM2_HC) TPM2_HT_TRANSIENT) << TPM2_HR_SHIFT)
+#define TPM2_HR_PERSISTENT        (((TPM2_HC) TPM2_HT_PERSISTENT) << TPM2_HR_SHIFT)
+#define TPM2_HR_NV_INDEX          (((TPM2_HC) TPM2_HT_NV_INDEX) << TPM2_HR_SHIFT)
+#define TPM2_HR_PERMANENT         (((TPM2_HC) TPM2_HT_PERMANENT) << TPM2_HR_SHIFT)
 #define TPM2_PCR_FIRST            ((TPM2_HC) (TPM2_HR_PCR + 0)) /* first PCR */
 #define TPM2_PCR_LAST             ((TPM2_HC) (TPM2_PCR_FIRST + TPM2_MAX_PCRS - 1)) /* last PCR */
 #define TPM2_HMAC_SESSION_FIRST   ((TPM2_HC) (TPM2_HR_HMAC_SESSION + 0)) /* first HMAC session */

src/tss2-esys/esys_crypto.c

@@ -636,7 +636,7 @@ iesys_crypto_KDFa(ESYS_CRYPTO_CALLBACKS *crypto_cb,
         return_if_error(r, "Error");
     }
     if ((bitLength % 8) != 0)
-        outKey[0] &= ((1 << (bitLength % 8)) - 1);
+        outKey[0] &= ((((BYTE)1) << (bitLength % 8)) - 1);
     if (counterInOut != NULL)
         *counterInOut = counter;
     LOGBLOB_DEBUG(outKey, (bitLength + 7) / 8, "IESYS KDFa key");
@@ -733,7 +733,7 @@ iesys_crypto_KDFe(ESYS_CRYPTO_CALLBACKS *crypto_cb,
         }
     LOGBLOB_DEBUG(key, bit_size/8, "Result KDFe");
     if((bit_size % 8) != 0)
-        key[0] &= ((1 << (bit_size % 8)) - 1);
+        key[0] &= ((((BYTE)1) << (bit_size % 8)) - 1);
     return r;
 
  error:

src/tss2-fapi/ifapi_helpers.c

@@ -2156,7 +2156,7 @@ ifapi_calculate_pcrs(
     for (i = 0; i < pcr_selection->count; i++) {
         for (pcr = 0; pcr < TPM2_MAX_PCRS; pcr++) {
             uint8_t byte_idx = pcr / 8;
-            uint8_t flag = 1 << (pcr % 8);
+            uint8_t flag = ((uint8_t)1) << (pcr % 8);
             if (flag & pcr_selection->pcrSelections[i].pcrSelect[byte_idx]) {
                 hash_size = ifapi_hash_get_digest_size(pcr_selection->pcrSelections[i].hash);
                 pcrs[n_pcrs].pcr = pcr;
@@ -2393,10 +2393,10 @@ ifapi_filter_pcr_selection_by_index(
     UINT8 selection[] = { 0, 0, 0, 0 };
 
     for (i = 0; i < pcr_count; i++) {
-        selection[0] |= (1 << pcr_index[i]) % 256;
-        selection[1] |= (1 << (pcr_index[i] - 8)) % 256;
-        selection[2] |= (1 << (pcr_index[i] - 16)) % 256;
-        selection[3] |= (1 << (pcr_index[i] - 24)) % 256;
+        selection[0] |= (((UINT32)1) << pcr_index[i]) % 256;
+        selection[1] |= (((UINT32)1) << (pcr_index[i] - 8)) % 256;
+        selection[2] |= (((UINT32)1) << (pcr_index[i] - 16)) % 256;
+        selection[3] |= (((UINT32)1) << (pcr_index[i] - 24)) % 256;
     };
 
     /* Remove unselected PCRs */
@@ -2494,7 +2494,7 @@ ifapi_compute_policy_digest(
         if (pcrIndex + 1 > max_pcr)
             max_pcr = pcrIndex + 1;
         pcr_selection->pcrSelections[j].pcrSelect[pcrIndex / 8] |=
-            1 << pcrIndex % 8;
+            ((BYTE)1) << pcrIndex % 8;
         if ((pcrIndex / 8) + 1 > pcr_selection->pcrSelections[j].sizeofSelect)
             pcr_selection->pcrSelections[j].sizeofSelect = (pcrIndex / 8) + 1;
     }
@@ -2517,7 +2517,7 @@ ifapi_compute_policy_digest(
                        hashAlg);
         }
         for (pcr = 0; pcr < max_pcr; pcr++) {
-            if ((selection.pcrSelect[pcr / 8]) & (1 << (pcr % 8))) {
+            if ((selection.pcrSelect[pcr / 8]) & (((BYTE)1) << (pcr % 8))) {
                 /* pcr selected */
                 for (j = 0; j < pcrs->count; j++) {
                     if (pcrs->pcrs[j].pcr == pcr) {
@@ -2648,7 +2648,7 @@ TSS2_RC ifapi_pcr_selection_to_pcrvalues(
     for (i = 0; i < pcr_selection->count; i++) {
         for (pcr = 0; pcr < TPM2_MAX_PCRS; pcr++) {
             uint8_t byte_idx = pcr / 8;
-            uint8_t flag = 1 << (pcr % 8);
+            uint8_t flag = ((uint8_t)1) << (pcr % 8);
             /* Check whether PCR is used. */
             if (flag & pcr_selection->pcrSelections[i].pcrSelect[byte_idx])
                 n_pcrs += 1;
@@ -2664,7 +2664,7 @@ TSS2_RC ifapi_pcr_selection_to_pcrvalues(
     for (i = 0; i < pcr_selection->count; i++) {
         for (pcr = 0; pcr < TPM2_MAX_PCRS; pcr++) {
             uint8_t byte_idx = pcr / 8;
-            uint8_t flag = 1 << (pcr % 8);
+            uint8_t flag = ((uint8_t)1) << (pcr % 8);
             /* Check whether PCR is used. */
             if (flag & pcr_selection->pcrSelections[i].pcrSelect[byte_idx]) {
                 pcr_values->pcrs[i_pcr].pcr = pcr;

src/tss2-fapi/ifapi_policy_callbacks.c

@@ -361,7 +361,7 @@ TSS2_RC ifapi_read_pcr(
             for (i = 0; i < profile_selection->count; i++) {
                 for (pcr = 0; pcr < TPM2_MAX_PCRS; pcr++) {
                     uint8_t byte_idx = pcr / 8;
-                    uint8_t flag = 1 << (pcr % 8);
+                    uint8_t flag = ((uint8_t)1) << (pcr % 8);
                     /* Check whether PCR is used. */
                     if ((flag & profile_selection->pcrSelections[i].pcrSelect[byte_idx]) &&
                         (flag & pcr_selection->selections.pcr_select.pcrSelect[byte_idx])) {

src/tss2-fapi/tpm_json_deserialize.c

@@ -320,7 +320,7 @@ ifapi_json_pcr_selection_deserialize(
         r = get_number_from_json(json_object_array_get_idx(jso, i), &n);
         return_if_error(r, "Bad PCR value");
         n_byte = n / 8;
-        pcrSelect[n_byte] |= (BYTE)(1 << (n % 8));
+        pcrSelect[n_byte] |= ((BYTE)1) << (n % 8);
         if (n_byte > *sizeofSelect)
             *sizeofSelect = n_byte;
     }

src/tss2-fapi/tpm_json_serialize.c

@@ -68,7 +68,7 @@ ifapi_json_pcr_select_serialize(
     json_object *jso2;
     for (i1 = 0; i1 < TPM2_PCR_LAST - TPM2_PCR_FIRST; i1++) {
         i2 = i1 + TPM2_PCR_FIRST;
-        if (pcrSelect[i2 / 8] & (BYTE)(1 << (i2 % 8))) {
+        if (pcrSelect[i2 / 8] & (((BYTE)1) << (i2 % 8))) {
             jso2 = json_object_new_int(i2);
             return_if_null(jso2, "Out of memory.", TSS2_FAPI_RC_MEMORY);
             if (json_object_array_add(*jso, jso2)) {

src/tss2-rc/tss2_rc.c

@@ -117,7 +117,7 @@ tss2_rc_layer_number_get(TSS2_RC rc)
 static inline UINT8
 tpm2_rc_fmt1_N_get(TPM2_RC rc)
 {
-    return ((rc & (0xF << 8)) >> 8);
+    return ((rc & (((TPM2_RC)0xF) << 8)) >> 8);
 }
 
 /**
@@ -151,7 +151,7 @@ tpm2_rc_fmt1_N_is_handle(TPM2_RC rc)
 static inline UINT8
 tpm2_rc_fmt1_P_get(TPM2_RC rc)
 {
-    return ((rc & (1 << 6)) >> 6);
+    return ((rc & (((TPM2_RC)1) << 6)) >> 6);
 }
 
 static inline UINT8
@@ -169,19 +169,19 @@ tpm2_rc_fmt0_error_get(TPM2_RC rc)
 static inline UINT8
 tpm2_rc_tpm_fmt0_V_get(TPM2_RC rc)
 {
-    return ((rc & (1 << 8)) >> 8);
+    return ((rc & (((TPM2_RC)1) << 8)) >> 8);
 }
 
 static inline UINT8
 tpm2_rc_fmt0_T_get(TPM2_RC rc)
 {
-    return ((rc & (1 << 10)) >> 8);
+    return ((rc & (((TPM2_RC)1) << 10)) >> 8);
 }
 
 static inline UINT8
 tpm2_rc_fmt0_S_get(TSS2_RC rc)
 {
-    return ((rc & (1 << 11)) >> 8);
+    return ((rc & (((TPM2_RC)1) << 11)) >> 8);
 }
 
 /**
@@ -696,7 +696,7 @@ tpm2_err_handler_fmt0(TSS2_RC rc)
 static inline UINT8
 tss2_rc_layer_format_get(TSS2_RC rc)
 {
-    return ((rc & (1 << 7)) >> 7);
+    return ((rc & (((TPM2_RC)1) << 7)) >> 7);
 }
 
 /**

src/tss2-tcti/mpsse/support.c

@@ -257,11 +257,11 @@ int gpio_write (struct mpsse_context *mpsse, int pin, int direction)
     {
         if (direction == HIGH)
         {
-            mpsse->bitbang |= (1 << pin);
+            mpsse->bitbang |= (((uint8_t)1) << pin);
         }
         else
         {
-            mpsse->bitbang &= ~(1 << pin);
+            mpsse->bitbang &= ~(((uint8_t)1) << pin);
         }
 
         if (set_bits_high (mpsse, mpsse->bitbang) == MPSSE_OK)
@@ -299,11 +299,11 @@ int gpio_write (struct mpsse_context *mpsse, int pin, int direction)
 
             if (direction == HIGH)
             {
-                mpsse->gpioh |= (1 << pin);
+                mpsse->gpioh |= (((uint8_t)1) << pin);
             }
             else
             {
-                mpsse->gpioh &= ~(1 << pin);
+                mpsse->gpioh &= ~(((uint8_t)1) << pin);
             }
             retval = set_bits_high (mpsse, mpsse->gpioh);
         }

src/util/tss2_endian.h

@@ -35,28 +35,28 @@
 
 static inline uint16_t endian_conv_16(uint16_t value)
 {
-    return ((value & (0xff))      << 8) | \
-           ((value & (0xff << 8)) >> 8);
+    return ((value & (((uint16_t)0xffU)))      << 8) | \
+           ((value & (((uint16_t)0xffU) << 8)) >> 8);
 }
 
 static inline uint32_t endian_conv_32(uint32_t value)
 {
-    return ((value & (0xff))       << 24) | \
-           ((value & (0xff << 8))  << 8)  | \
-           ((value & (0xff << 16)) >> 8)  | \
-           ((value & (0xff << 24)) >> 24);
+    return ((value & (((uint32_t)0xffU)))       << 24) | \
+           ((value & (((uint32_t)0xffU) << 8))  << 8)  | \
+           ((value & (((uint32_t)0xffU) << 16)) >> 8)  | \
+           ((value & (((uint32_t)0xffU) << 24)) >> 24);
 }
 
 static inline uint64_t endian_conv_64(uint64_t value)
 {
-    return ((value & (0xffULL))       << 56) | \
-           ((value & (0xffULL << 8))  << 40) | \
-           ((value & (0xffULL << 16)) << 24) | \
-           ((value & (0xffULL << 24)) << 8)  | \
-           ((value & (0xffULL << 32)) >> 8)  | \
-           ((value & (0xffULL << 40)) >> 24) | \
-           ((value & (0xffULL << 48)) >> 40) | \
-           ((value & (0xffULL << 56)) >> 56);
+    return ((value & (((uint64_t)0xffU)))       << 56) | \
+           ((value & (((uint64_t)0xffU) << 8))  << 40) | \
+           ((value & (((uint64_t)0xffU) << 16)) << 24) | \
+           ((value & (((uint64_t)0xffU) << 24)) << 8)  | \
+           ((value & (((uint64_t)0xffU) << 32)) >> 8)  | \
+           ((value & (((uint64_t)0xffU) << 40)) >> 24) | \
+           ((value & (((uint64_t)0xffU) << 48)) >> 40) | \
+           ((value & (((uint64_t)0xffU) << 56)) >> 56);
 }
 
 #if defined(WORDS_BIGENDIAN)