Merge pull request planetbeing#6 from bhearsum/attributable

Add support for extended attributes and quick attribution of DMGs

.gitignore

@@ -1 +1,3 @@
 build
+/venv/
+/test/*.t.err

README.markdown

@@ -73,4 +73,3 @@ The makefile in the root folder will make all utilities.
 ### hdutil
 	cd hdiutil
 	make
-

dmg/CMakeLists.txt

@@ -1,4 +1,4 @@
-INCLUDE(FindOpenSSL)
+# INCLUDE(FindOpenSSL)
 INCLUDE(FindZLIB)
 INCLUDE(FindBZip2)
 
@@ -16,7 +16,7 @@ link_directories(${BZIP2_LIBRARIES})
 
 link_directories(${PROJECT_BINARY_DIR}/common ${PROJECT_BINARY_DIR}/hfs)
 
-add_library(dmg adc.c checksum.c dmgfile.c dmglib.c filevault.c io.c partition.c resources.c udif.c ../includes/dmg/adc.h ../includes/dmg/dmg.h ../includes/dmg/dmgfile.h ../includes/dmg/dmglib.h ../includes/dmg/filevault.h)
+add_library(dmg adc.c attribution.c checksum.c dmgfile.c dmglib.c filevault.c io.c partition.c resources.c udif.c ../includes/dmg/adc.h ../includes/dmg/attribution.h ../includes/dmg/dmg.h ../includes/dmg/dmgfile.h ../includes/dmg/dmglib.h ../includes/dmg/filevault.h)
 
 IF(OPENSSL_FOUND)
 	add_definitions(-DHAVE_CRYPT)

dmg/dmg.c

@@ -22,7 +22,7 @@ int buildInOut(const char* source, const char* dest, AbstractFile** in, Abstract
 		return FALSE;
 	}
 
-	*out = createAbstractFileFromFile(fopen(dest, "wb"));
+	*out = createAbstractFileFromFile(strcmp(dest, "-") == 0 ? stdout : fopen(dest, "wb"));
 	if(!(*out)) {
 		(*in)->close(*in);
 		printf("cannot open destination: %s\n", dest);
@@ -41,7 +41,7 @@ int main(int argc, char* argv[]) {
 	TestByteOrder();
 
 	if(argc < 4) {
-		printf("usage: %s [extract|build|build2048|res|iso|dmg] <in> <out> (-k <key>) (partition)\n", argv[0]);
+		printf("usage: %s [extract|build|build2048|res|iso|dmg|attribute] <in> <out> (-k <key>) (partition)\n", argv[0]);
 		return 0;
 	}
 
@@ -72,16 +72,21 @@ int main(int argc, char* argv[]) {
 		}
 		extractDmg(in, out, partNum);
 	} else if(strcmp(argv[1], "build") == 0) {
-		buildDmg(in, out, SECTOR_SIZE);
+		buildDmg(in, out, SECTOR_SIZE, argc > 4 ? argv[4] : NULL);
 	} else if(strcmp(argv[1], "build2048") == 0) {
-		buildDmg(in, out, 2048);
+		buildDmg(in, out, 2048, NULL);
 	} else if(strcmp(argv[1], "res") == 0) {
 		outResources(in, out);
 	} else if(strcmp(argv[1], "iso") == 0) {
 		convertToISO(in, out);
 	} else if(strcmp(argv[1], "dmg") == 0) {
 		convertToDMG(in, out);
+	} else if(strcmp(argv[1], "attribute") == 0) {
+		if(argc < 6) {
+			printf("Not enough arguments: attribute <in> <out> <sentinel> <string>");
+		}
+		updateAttribution(in, out, argv[4], argv[5], strlen(argv[5]));
 	}
-	
+
 	return 0;
 }

dmg/dmgfile.c

@@ -209,7 +209,7 @@ io_func* openDmgFile(AbstractFile* abstractIn) {
     ASSERT( abstractIn->read(abstractIn, dmg->resourceXML, (size_t)dmg->resourceFile.fUDIFXMLLength) == (size_t)dmg->resourceFile.fUDIFXMLLength, "fread" );
     dmg->resourceXML[dmg->resourceFile.fUDIFXMLLength] = 0;
 
-	dmg->resources = readResources(dmg->resourceXML, dmg->resourceFile.fUDIFXMLLength);
+	dmg->resources = readResources(dmg->resourceXML, dmg->resourceFile.fUDIFXMLLength, true);
 	dmg->numBLKX = 0;
 
 	blkx = (getResourceByKey(dmg->resources, "blkx"))->data;

dmg/dmglib.c

@@ -1,6 +1,7 @@
 #include <string.h>
 #include "common.h"
 #include "abstractfile.h"
+#include <dmg/attribution.h>
 #include <dmg/dmg.h>
 #include <dmg/dmgfile.h>
 
@@ -87,7 +88,7 @@ uint32_t calculateMasterChecksum(ResourceKey* resources) {
 	return result;  
 }
 
-int buildDmg(AbstractFile* abstractIn, AbstractFile* abstractOut, unsigned int BlockSize) {	
+int buildDmg(AbstractFile* abstractIn, AbstractFile* abstractOut, unsigned int BlockSize, const char* sentinel) {
 	io_func* io;
 	Volume* volume;  
 
@@ -147,15 +148,32 @@ int buildDmg(AbstractFile* abstractIn, AbstractFile* abstractOut, unsigned int B
 	printf("Writing main data blkx...\n"); fflush(stdout);
 
 	abstractIn->seek(abstractIn, 0);
+
+	AbstractAttribution* attribution = NULL;
+	if (sentinel) {
+		attribution = createAbstractAttributionPreservingSentinel(sentinel);
+	}
+
+	if (attribution) {
+		attribution->beforeMainBlkx(attribution, abstractOut, &dataForkToken);
+	}
+
 	blkx = insertBLKX(abstractOut, abstractIn, USER_OFFSET, (volumeHeader->totalBlocks * volumeHeader->blockSize)/SECTOR_SIZE,
-				pNum, CHECKSUM_UDIF_CRC32, &BlockSHA1CRC, &uncompressedToken, &CRCProxy, &dataForkToken, volume, 0);
+				pNum, CHECKSUM_UDIF_CRC32, &BlockSHA1CRC, &uncompressedToken, &CRCProxy, &dataForkToken, volume, 0, attribution);
+
+	AttributionResource attributionResource;
+	memset(&attributionResource, 0, sizeof(AttributionResource));
+
+	if (attribution) {
+		attribution->afterMainBlkx(attribution, abstractOut, &dataForkToken, &attributionResource);
+	}
 
 	blkx->checksum.data[0] = uncompressedToken.crc;
 	printf("Inserting main blkx...\n"); fflush(stdout);
 
 	char pName[100];
 	sprintf(pName, "Mac_OS_X (Apple_HFSX : %d)", pNum + 1);	
-	resources = insertData(resources, "blkx", pNum, pName, (const char*) blkx, sizeof(BLKXTable) + (blkx->blocksRunCount * sizeof(BLKXRun)), ATTRIBUTE_HDIUTIL);
+	resources = insertData(resources, "blkx", pNum, pName, 0, false, (const char*) blkx, sizeof(BLKXTable) + (blkx->blocksRunCount * sizeof(BLKXRun)), ATTRIBUTE_HDIUTIL);
 	free(blkx);
 
 	printf("Inserting cSum data...\n"); fflush(stdout);
@@ -164,7 +182,7 @@ int buildDmg(AbstractFile* abstractIn, AbstractFile* abstractOut, unsigned int B
 	csum.type = CHECKSUM_MKBLOCK;
 	csum.checksum = uncompressedToken.block;
 
-	resources = insertData(resources, "cSum", 2, "", (const char*) (&csum), sizeof(csum), 0);
+	resources = insertData(resources, "cSum", 2, "", 0, false, (const char*) (&csum), sizeof(csum), 0);
 
 	printf("Inserting nsiz data\n"); fflush(stdout);
 
@@ -209,14 +227,14 @@ int buildDmg(AbstractFile* abstractIn, AbstractFile* abstractOut, unsigned int B
 	curResource = curResource->next;
 	releaseNSiz(nsiz);
 
-	curResource->next = makePlst();
+	curResource->next = makePlst((const char*) (&attributionResource), sizeof(attributionResource), true);
 	curResource = curResource->next;
 
 	curResource->next = makeSize(volumeHeader);
 	curResource = curResource->next;
 
 	plistOffset = abstractOut->tell(abstractOut);
-	writeResources(abstractOut, resources);
+	writeResources(abstractOut, resources, true);
 	plistSize = abstractOut->tell(abstractOut) - plistOffset;
 
 	printf("Generating UDIF metadata...\n"); fflush(stdout);
@@ -353,17 +371,17 @@ int convertToDMG(AbstractFile* abstractIn, AbstractFile* abstractOut) {
 
 			abstractIn->seek(abstractIn, partitions[i].pmPyPartStart * BlockSize);
 			blkx = insertBLKX(abstractOut, abstractIn, partitions[i].pmPyPartStart, partitions[i].pmPartBlkCnt, i, CHECKSUM_UDIF_CRC32,
-						&BlockCRC, &uncompressedToken, &CRCProxy, &dataForkToken, NULL, 0);
+						&BlockCRC, &uncompressedToken, &CRCProxy, &dataForkToken, NULL, 0, NULL);
 
 			blkx->checksum.data[0] = uncompressedToken.crc;	
-			resources = insertData(resources, "blkx", i, partitionName, (const char*) blkx, sizeof(BLKXTable) + (blkx->blocksRunCount * sizeof(BLKXRun)), ATTRIBUTE_HDIUTIL);
+			resources = insertData(resources, "blkx", i, partitionName, 0, false, (const char*) blkx, sizeof(BLKXTable) + (blkx->blocksRunCount * sizeof(BLKXRun)), ATTRIBUTE_HDIUTIL);
 			free(blkx);
 
 			memset(&csum, 0, sizeof(CSumResource));
 			csum.version = 1;
 			csum.type = CHECKSUM_MKBLOCK;
 			csum.checksum = uncompressedToken.block;
-			resources = insertData(resources, "cSum", i, "", (const char*) (&csum), sizeof(csum), 0);
+			resources = insertData(resources, "cSum", i, "", 0, false, (const char*) (&csum), sizeof(csum), 0);
 
 			if(nsiz == NULL) {
 				nsiz = myNSiz = (NSizResource*) malloc(sizeof(NSizResource));
@@ -394,16 +412,16 @@ int convertToDMG(AbstractFile* abstractIn, AbstractFile* abstractOut) {
 
 		abstractIn->seek(abstractIn, 0);
 		blkx = insertBLKX(abstractOut, abstractIn, 0, fileLength/SECTOR_SIZE, ENTIRE_DEVICE_DESCRIPTOR, CHECKSUM_UDIF_CRC32,
-					&BlockCRC, &uncompressedToken, &CRCProxy, &dataForkToken, NULL, 0);
+					&BlockCRC, &uncompressedToken, &CRCProxy, &dataForkToken, NULL, 0, NULL);
 		blkx->checksum.data[0] = uncompressedToken.crc;
-		resources = insertData(resources, "blkx", 0, "whole disk (unknown partition : 0)", (const char*) blkx, sizeof(BLKXTable) + (blkx->blocksRunCount * sizeof(BLKXRun)), ATTRIBUTE_HDIUTIL);
+		resources = insertData(resources, "blkx", 0, "whole disk (unknown partition : 0)", 0, false, (const char*) blkx, sizeof(BLKXTable) + (blkx->blocksRunCount * sizeof(BLKXRun)), ATTRIBUTE_HDIUTIL);
 		free(blkx);
 
 		memset(&csum, 0, sizeof(CSumResource));
 		csum.version = 1;
 		csum.type = CHECKSUM_MKBLOCK;
 		csum.checksum = uncompressedToken.block;
-		resources = insertData(resources, "cSum", 0, "", (const char*) (&csum), sizeof(csum), 0);
+		resources = insertData(resources, "cSum", 0, "", 0, false, (const char*) (&csum), sizeof(csum), 0);
 
 		if(nsiz == NULL) {
 			nsiz = myNSiz = (NSizResource*) malloc(sizeof(NSizResource));
@@ -433,11 +451,13 @@ int convertToDMG(AbstractFile* abstractIn, AbstractFile* abstractOut) {
 	curResource = curResource->next;
 	releaseNSiz(nsiz);
 
-	curResource->next = makePlst();
+	curResource->next = makePlst("", 0, false);
 	curResource = curResource->next;
 
 	plistOffset = abstractOut->tell(abstractOut);
-	writeResources(abstractOut, resources);
+	// Note: Passing false here means that attribution data is not preserved through
+	// a `dmg convert` operation.
+	writeResources(abstractOut, resources, false);
 	plistSize = abstractOut->tell(abstractOut) - plistOffset;
 
 	printf("Generating UDIF metadata...\n"); fflush(stdout);

dmg/io.c

@@ -6,6 +6,7 @@
 
 #include <dmg/dmg.h>
 #include <dmg/adc.h>
+#include <dmg/attribution.h>
 #include <inttypes.h>
 
 #define SECTORS_AT_A_TIME 0x200
@@ -23,14 +24,15 @@
 
 BLKXTable* insertBLKX(AbstractFile* out, AbstractFile* in, uint32_t firstSectorNumber, uint32_t numSectors, uint32_t blocksDescriptor,
 			uint32_t checksumType, ChecksumFunc uncompressedChk, void* uncompressedChkToken, ChecksumFunc compressedChk,
-			void* compressedChkToken, Volume* volume, int addComment) {
+			void* compressedChkToken, Volume* volume, int addComment, AbstractAttribution* attribution) {
 	BLKXTable* blkx;
 
 	uint32_t roomForRuns;
 	uint32_t curRun;
 	uint64_t curSector;
 
 	unsigned char* inBuffer;
+	unsigned char* nextInBuffer;
 	unsigned char* outBuffer;
 	size_t bufferSize;
 	size_t have;
@@ -65,13 +67,20 @@ BLKXTable* insertBLKX(AbstractFile* out, AbstractFile* in, uint32_t firstSectorN
 	bufferSize = SECTOR_SIZE * blkx->decompressBufferRequested;
 
 	ASSERT(inBuffer = (unsigned char*) malloc(bufferSize), "malloc");
+	ASSERT(nextInBuffer = (unsigned char*) malloc(bufferSize), "malloc");
 	ASSERT(outBuffer = (unsigned char*) malloc(bufferSize), "malloc");
 
 	curRun = 0;
 	curSector = 0;
 
 	uint64_t startOff = in->tell(in);
 
+	enum ShouldKeepRaw keepRaw = KeepNoneRaw;
+	// We never want the iOS-specific tweaks when building attributable DMGs.
+	if (attribution) {
+		ASSERT(!addComment, "No attribution with addComment!");
+	}
+
 	if(addComment)
 	{
 		blkx->runs[curRun].type = BLOCK_COMMENT;
@@ -93,7 +102,6 @@ BLKXTable* insertBLKX(AbstractFile* out, AbstractFile* in, uint32_t firstSectorN
 			roomForRuns <<= 1;
 			blkx = (BLKXTable*) realloc(blkx, sizeof(BLKXTable) + (roomForRuns * sizeof(BLKXRun)));
 		}
-
 		blkx->runs[curRun].type = BLOCK_BZIP2;
 		blkx->runs[curRun].reserved = 0;
 		blkx->runs[curRun].sectorStart = curSector;
@@ -104,7 +112,8 @@ BLKXTable* insertBLKX(AbstractFile* out, AbstractFile* in, uint32_t firstSectorN
 		strm.bzfree = Z_NULL;
 		strm.opaque = Z_NULL;
 
-		int amountRead;
+		int amountRead = 0;
+		int nextAmountRead = 0;
 		{
 			size_t sectorsToSkip = 0;
 			size_t processed = 0;
@@ -114,9 +123,16 @@ BLKXTable* insertBLKX(AbstractFile* out, AbstractFile* in, uint32_t firstSectorN
 				blkx->runs[curRun].sectorCount = ((numSectors - processed) > SECTORS_AT_A_TIME) ? SECTORS_AT_A_TIME : (numSectors - processed);
 
 				//printf("Currently at %" PRId64 "\n", curOff);
-				in->seek(in, startOff + (blkx->sectorCount - numSectors + processed) * SECTOR_SIZE);
+				off_t sectorStart = startOff + (blkx->sectorCount - numSectors + processed) * SECTOR_SIZE;
+				in->seek(in, sectorStart);
 				ASSERT((amountRead = in->read(in, inBuffer, blkx->runs[curRun].sectorCount * SECTOR_SIZE)) == (blkx->runs[curRun].sectorCount * SECTOR_SIZE), "mRead");
 
+				if (numSectors - blkx->runs[curRun].sectorCount > 0) {
+					// No need to rewind `inBuffer` because the next iteration of the loop
+					// calls `seek` anyways.
+					nextAmountRead = in->read(in, nextInBuffer, blkx->runs[curRun].sectorCount * SECTOR_SIZE);
+				}
+
 				if(!addComment)
 					break;
 
@@ -202,6 +218,27 @@ BLKXTable* insertBLKX(AbstractFile* out, AbstractFile* in, uint32_t firstSectorN
 		strm.avail_in = amountRead;
 		strm.next_in = (char*)inBuffer;
 
+		if (attribution) {
+			// We either haven't found the sentinel value yet, or are already past it.
+			// Either way, keep searching.
+			if (keepRaw == KeepNoneRaw) {
+				keepRaw = attribution->shouldKeepRaw(attribution, inBuffer, amountRead, nextInBuffer, nextAmountRead);
+			}
+			// KeepCurrentAndNextRaw means that the *previous* time through the loop `shouldKeepRaw`
+			// found the sentinel string, and that it crosses two runs. The previous
+			// loop already kept its run raw, and so must we. We don't want the _next_ run 
+			// to also be raw though, so we adjust this appropriately.
+			// Note that KeepCurrentRaw will switch to KeepNoneRaw further down, when we've
+			// set the run raw.
+			else if (keepRaw == KeepCurrentAndNextRaw) {
+				keepRaw = KeepCurrentRaw;
+			}
+			else if (keepRaw == KeepCurrentRaw) {
+				keepRaw = KeepRemainingRaw;
+			}
+			printf("keepRaw = %d (%p, %d)\n", keepRaw, inBuffer, amountRead);
+		}
+
 		if(uncompressedChk)
 			(*uncompressedChk)(uncompressedChkToken, inBuffer, blkx->runs[curRun].sectorCount * SECTOR_SIZE);
 
@@ -217,20 +254,35 @@ BLKXTable* insertBLKX(AbstractFile* out, AbstractFile* in, uint32_t firstSectorN
 		}
 		have = bufferSize - strm.avail_out;
 
-		if((have / SECTOR_SIZE) >= (blkx->runs[curRun].sectorCount - 15)) {
+		if(keepRaw == KeepCurrentRaw || keepRaw == KeepCurrentAndNextRaw || ((have / SECTOR_SIZE) >= (blkx->runs[curRun].sectorCount - 15))) {
+			printf("Setting type = BLOCK_RAW\n");
 			blkx->runs[curRun].type = BLOCK_RAW;
 			ASSERT(out->write(out, inBuffer, blkx->runs[curRun].sectorCount * SECTOR_SIZE) == (blkx->runs[curRun].sectorCount * SECTOR_SIZE), "fwrite");
 			blkx->runs[curRun].compLength += blkx->runs[curRun].sectorCount * SECTOR_SIZE;
 
+
 			if(compressedChk)
 				(*compressedChk)(compressedChkToken, inBuffer, blkx->runs[curRun].sectorCount * SECTOR_SIZE);
 
+			if (attribution) {
+				// In a raw block, uncompressed and compressed data is identical.
+				attribution->observeBuffers(attribution, keepRaw,
+											inBuffer, blkx->runs[curRun].sectorCount * SECTOR_SIZE,
+											inBuffer, blkx->runs[curRun].sectorCount * SECTOR_SIZE);
+			}
 		} else {
 			ASSERT(out->write(out, outBuffer, have) == have, "fwrite");
 
 			if(compressedChk)
 				(*compressedChk)(compressedChkToken, outBuffer, have);
 
+			if (attribution) {
+				// In a bzip2 block, uncompressed and compressed data are not the same.
+				attribution->observeBuffers(attribution, keepRaw,
+											inBuffer, amountRead,
+											outBuffer, have);
+			}
+
 			blkx->runs[curRun].compLength += have;
 		}