libselinux: correctly hash specfiles larger than 4G

The internal Sha1Update() functions only handles buffers up to a size of
UINT32_MAX, due to its usage of the type uint32_t.  This causes issues
when processing more than UINT32_MAX bytes, e.g. with a specfile larger
than 4G.  0aa974a ("libselinux: limit has buffer size") tried to
address this issue, but failed since the overflow check

    if (digest->hashbuf_size + buf_len < digest->hashbuf_size) {

will be done in the widest common type, which is size_t, the type of
`buf_len`.

Revert the type of `hashbuf_size` to size_t and instead process the data
in blocks of supported size.

Acked-by: James Carter <jwcart2@gmail.com>
Signed-off-by: Christian Göttsche <cgzones@googlemail.com>
Reverts: 0aa974a ("libselinux: limit has buffer size")

libselinux/src/label_internal.h

@@ -57,7 +57,7 @@ int selabel_service_init(struct selabel_handle *rec,
 struct selabel_digest {
 	unsigned char *digest;	/* SHA1 digest of specfiles */
 	unsigned char *hashbuf;	/* buffer to hold specfiles */
-	uint32_t hashbuf_size;	/* buffer size */
+	size_t hashbuf_size;	/* buffer size */
 	size_t specfile_cnt;	/* how many specfiles processed */
 	char **specfile_list;	/* and their names */
 };

libselinux/src/label_support.c

@@ -116,13 +116,25 @@ int  read_spec_entries(char *line_buf, const char **errbuf, int num_args, ...)
 void  digest_gen_hash(struct selabel_digest *digest)
 {
 	Sha1Context context;
+	size_t remaining_size;
+	const unsigned char *ptr;
 
 	/* If SELABEL_OPT_DIGEST not set then just return */
 	if (!digest)
 		return;
 
 	Sha1Initialise(&context);
-	Sha1Update(&context, digest->hashbuf, digest->hashbuf_size);
+
+	/* Process in blocks of UINT32_MAX bytes */
+	remaining_size = digest->hashbuf_size;
+	ptr = digest->hashbuf;
+	while (remaining_size > UINT32_MAX) {
+		Sha1Update(&context, ptr, UINT32_MAX);
+		remaining_size -= UINT32_MAX;
+		ptr += UINT32_MAX;
+	}
+	Sha1Update(&context, ptr, remaining_size);
+
 	Sha1Finalise(&context, (SHA1_HASH *)digest->digest);
 	free(digest->hashbuf);
 	digest->hashbuf = NULL;