diff --git a/apple/header_map.bzl b/apple/header_map.bzl new file mode 100644 index 0000000000..00c89a7a01 --- /dev/null +++ b/apple/header_map.bzl @@ -0,0 +1,24 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Rules for creating header maps. +""" + +load( + "@build_bazel_rules_apple//apple/internal:header_map.bzl", + _header_map = "header_map", +) + +header_map = _header_map diff --git a/apple/internal/header_map.bzl b/apple/internal/header_map.bzl new file mode 100644 index 0000000000..909f9f4c5b --- /dev/null +++ b/apple/internal/header_map.bzl @@ -0,0 +1,129 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Rule for creating header_maps.""" + +load("@build_bazel_rules_swift//swift:swift.bzl", "swift_common") + +HeaderMapInfo = provider( + doc = "Provides information about created `.hmap` (header map) files", + fields = { + "files": "depset of paths of any header_maps", + }, +) + +def _write_header_map(actions, header_map_tool, output, namespace, hdrs_lists): + """Makes a binary hmap file by executing the header_map tool. + + Args: + actions: a ctx.actions struct + header_map_tool: an executable pointing to @bazel_build_rules_ios//rules/hmap:hmaptool + output: the output file that will contain the built hmap + namespace: the prefix to be used for header imports + hdrs_lists: an array of enumerables containing headers to be added to the hmap + """ + + args = actions.args() + if namespace: + args.add("--namespace", namespace) + + args.add("--output", output) + + for hdrs in hdrs_lists: + args.add_all(hdrs) + + args.set_param_file_format(format = "multiline") + args.use_param_file("@%s") + + actions.run( + mnemonic = "HmapWrite", + arguments = [args], + executable = header_map_tool, + outputs = [output], + ) + +def _header_map_impl(ctx): + """Implementation of the header_map() rule. + + It creates a text file with mappings and creates an action that calls out to the header_map tool + to convert it to a binary header_map file. + """ + hdrs_lists = [ctx.files.hdrs] if ctx.files.hdrs else [] + + for dep in ctx.attr.deps: + found_headers = [] + if apple_common.Objc in dep: + found_headers.append(getattr(dep[apple_common.Objc], "direct_headers", [])) + if CcInfo in dep: + found_headers.append(dep[CcInfo].compilation_context.direct_headers) + if not found_headers: + fail("Direct header provider: '%s' listed in 'deps' does not have any direct headers to provide." % dep) + hdrs_lists.extend(found_headers) + + hdrs_lists = [[h for h in hdrs if h.basename.endswith(".h")] for hdrs in hdrs_lists] + + _write_header_map( + actions = ctx.actions, + header_map_tool = ctx.executable._hmaptool, + output = ctx.outputs.header_map, + namespace = ctx.attr.namespace, + hdrs_lists = hdrs_lists, + ) + + return [ + apple_common.new_objc_provider(), + swift_common.create_swift_info(), + CcInfo( + compilation_context = cc_common.create_compilation_context( + headers = depset([ctx.outputs.header_map]), + ), + ), + HeaderMapInfo( + files = depset([ctx.outputs.header_map]), + ), + ] + +header_map = rule( + implementation = _header_map_impl, + output_to_genfiles = True, + attrs = { + "namespace": attr.string( + mandatory = False, + doc = "The prefix to be used for header imports", + ), + "hdrs": attr.label_list( + mandatory = False, + allow_files = True, + doc = "The list of headers included in the header_map", + ), + "deps": attr.label_list( + mandatory = False, + providers = [[apple_common.Objc], [CcInfo]], + doc = "Targets whose direct headers should be added to the list of hdrs", + ), + "_hmaptool": attr.label( + executable = True, + cfg = "exec", + default = Label("//tools/hmaptool:hmaptool"), + ), + }, + outputs = { + "header_map": "%{name}.hmap", + }, + doc = """\ +Creates a binary header_map file from the given headers suitable for passing to clang. + +This can be used to allow headers to be imported at a consistent path regardless of the package structure being used. + """, +) diff --git a/apple/internal/header_map_support.bzl b/apple/internal/header_map_support.bzl new file mode 100644 index 0000000000..6320b45ef7 --- /dev/null +++ b/apple/internal/header_map_support.bzl @@ -0,0 +1,99 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Provides utilities for working with header_maps within macros and rules""" + +load("//apple/internal:header_map.bzl", "header_map") + +def _create_header_map_context( + name, + module_name, + hdrs, + private_hdrs, + deps): + """ + Creates header_map(s) for the target with the given name. + + Args: + name: The name of the target. + module_name: The name of the module to use in the header map. + hdrs: The public headers to include in the header map. + private_hdrs: The private headers to include in the header map. + deps: The dependencies to include in the header map. + + Returns a struct (or `None` if no headers) with the following attributes: + objc_copts: The compiler options to use when compiling an Objective-C library with the header map. + swift_copts: The compiler options to use when compiling a Swift library with the header map. + header_maps: The labels to any generated header maps, these should be the inputs used with the `copts`. + """ + + header_maps = [] + copts = [] + + if hdrs: + public_hmap_name = name + ".public.hmap" + public_hdrs_filegroup = name + ".public.hmap_hdrs" + native.filegroup( + name = public_hdrs_filegroup, + srcs = hdrs, + ) + header_map( + name = public_hmap_name, + namespace = module_name, + hdrs = [public_hdrs_filegroup], + deps = deps, + ) + header_maps.append(":{}".format(public_hmap_name)) + copts.append("-I$(execpath :{})".format(public_hmap_name)) + copts.append("-I.") + + if private_hdrs: + private_hmap_name = name + ".private.hmap" + private_hdrs_filegroup = name + ".private_hdrs" + native.filegroup( + name = private_hdrs_filegroup, + srcs = private_hdrs, + ) + header_map( + name = private_hmap_name, + namespace = module_name, + hdrs = [private_hdrs_filegroup], + ) + header_maps.extend([ + ":{}".format(private_hmap_name), + ":{}".format(private_hdrs_filegroup), + ]) + copts.extend([ + "-I$(execpath :{})".format(private_hmap_name), + "-I$(execpath :{})".format(private_hmap_name), + "-iquote", + "$(execpath :{})".format(private_hmap_name), + ]) + + if not header_maps: + return None + + swift_copts = [] + for copt in copts: + swift_copts.extend(["-Xcc", copt]) + + return struct( + objc_copts = copts, + swift_copts = swift_copts, + header_maps = header_maps, + ) + +header_map_support = struct( + create_header_map_context = _create_header_map_context, +) diff --git a/examples/multi_platform/MixedLib/BUILD b/examples/multi_platform/MixedLib/BUILD index fc5e9077ee..f57f4b2845 100644 --- a/examples/multi_platform/MixedLib/BUILD +++ b/examples/multi_platform/MixedLib/BUILD @@ -3,6 +3,7 @@ load( "swift_library", ) load("//apple:apple.bzl", "experimental_mixed_language_library") +load("//apple:header_map.bzl", "header_map") load("//apple:ios.bzl", "ios_unit_test") experimental_mixed_language_library( @@ -11,8 +12,21 @@ experimental_mixed_language_library( "MixedAnswer.m", "MixedAnswer.swift", ], - hdrs = ["MixedAnswer.h"], + hdrs = [ + "MixedAnswer.h", + ":MixedAnswerHeaderMap", + ], enable_modules = True, + objc_copts = [ + "-I$(execpath :MixedAnswerHeaderMap)", + "-I.", + ], +) + +header_map( + name = "MixedAnswerHeaderMap", + hdrs = ["MixedAnswer.h"], + namespace = "MixedAnswer", ) swift_library( diff --git a/examples/multi_platform/MixedLib/MixedAnswer.m b/examples/multi_platform/MixedLib/MixedAnswer.m index 5bc6455a14..a5abe57df0 100644 --- a/examples/multi_platform/MixedLib/MixedAnswer.m +++ b/examples/multi_platform/MixedLib/MixedAnswer.m @@ -1,4 +1,5 @@ -#import "examples/multi_platform/MixedLib/MixedAnswer.h" +#import + #import "examples/multi_platform/MixedLib/MixedAnswer-Swift.h" @implementation MixedAnswerObjc diff --git a/test/starlark_tests/targets_under_test/apple/BUILD b/test/starlark_tests/targets_under_test/apple/BUILD index 80db21c1e3..f3f0fd9716 100644 --- a/test/starlark_tests/targets_under_test/apple/BUILD +++ b/test/starlark_tests/targets_under_test/apple/BUILD @@ -42,6 +42,7 @@ load( "dummy_test_runner", ) load("@bazel_skylib//rules:write_file.bzl", "write_file") +load("//apple:header_map.bzl", "header_map") licenses(["notice"]) @@ -1405,3 +1406,50 @@ swift_apple_core_ml_library( mlmodel = "//test/testdata/resources:sample.mlmodel", tags = common.fixture_tags, ) + +# --------------------------------------------------------------------------------------- +# Targets for header map rule tests. + +objc_library( + name = "header_map_objc_lib", + srcs = [ + "//test/starlark_tests/resources:shared.m", + ], + hdrs = [ + "//test/starlark_tests/resources:shared.h", + ], + tags = common.fixture_tags, +) + +swift_library( + name = "header_map_swift_lib", + srcs = ["DummyFmwk.swift"], + generates_header = True, + tags = common.fixture_tags, +) + +header_map( + name = "header_map_with_header", + hdrs = [ + "//test/starlark_tests/resources:shared.h", + ], + tags = common.fixture_tags, +) + +header_map( + name = "header_map_with_objc_lib_dep", + hdrs = [], + tags = common.fixture_tags, + deps = [ + ":header_map_objc_lib", + ], +) + +header_map( + name = "header_map_with_swift_lib_dep", + hdrs = [], + tags = common.fixture_tags, + deps = [ + ":header_map_swift_lib", + ], +) diff --git a/tools/hmaptool/BUILD b/tools/hmaptool/BUILD new file mode 100644 index 0000000000..c86996bcdb --- /dev/null +++ b/tools/hmaptool/BUILD @@ -0,0 +1,38 @@ +package(default_visibility = ["//apple/internal:__subpackages__"]) + +HMAP_COPTS = [ + "-DHASH_FUNCTION=HASH_MUR", + "-DHASH_USING_NO_STRICT_ALIASING", + "-fno-strict-aliasing", +] + +cc_library( + name = "lines", + srcs = ["lines.c"], + hdrs = ["lines.h"], + copts = ["-Wno-parentheses"], +) + +cc_library( + name = "hmap", + srcs = ["hmap.c"], + hdrs = ["hmap.h"], + copts = HMAP_COPTS, +) + +cc_binary( + name = "hmaptool", + srcs = [ + "hmaptool.c", + "uthash.h", + ], + copts = HMAP_COPTS, + # Used by the rule implementations, so it needs to be public; but + # should be considered an implementation detail of the rules and + # not used by other things. + visibility = ["//visibility:public"], + deps = [ + ":hmap", + ":lines", + ], +) diff --git a/tools/hmaptool/README.md b/tools/hmaptool/README.md new file mode 100644 index 0000000000..63e1db7c23 --- /dev/null +++ b/tools/hmaptool/README.md @@ -0,0 +1,9 @@ +# hmaptool + +This is a tool which creates binary headermaps to enable angle bracket imports of headers when building with Bazel. +Required to mimic Xcode's behavior of generating headermaps by default which iOS projects rely on. + +It includes the use of two C libraries from different projects: + +- [`lines.h/lines.c`](https://github.com/wscott/bksupport/tree/master/lines) +- [`uthash.h`](https://github.com/troydhanson/uthash/blob/v2.1.0/src/uthash.h) diff --git a/tools/hmaptool/hmap.c b/tools/hmaptool/hmap.c new file mode 100644 index 0000000000..516263b40a --- /dev/null +++ b/tools/hmaptool/hmap.c @@ -0,0 +1,435 @@ +#include "hmap.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Mac OS X doesn't have byteswap.h +#if defined(__APPLE__) +#include +#define bswap_16(x) OSSwapInt16(x) +#define bswap_32(x) OSSwapInt32(x) +#define bswap_64(x) OSSwapInt64(x) +#endif + +#define max(a, b) \ + ({ \ + __typeof__(a) _a = (a); \ + __typeof__(b) _b = (b); \ + _a > _b ? _a : _b; \ + }) + +enum { + HMAP_HeaderMagicNumber = ('h' << 24) | ('m' << 16) | ('a' << 8) | 'p', + HMAP_HeaderVersion = 1, + HMAP_EmptyBucketKey = 0 +}; + +typedef struct HMapBucket { + uint32_t Key; // Offset (into strings) of key. + uint32_t Prefix; // Offset (into strings) of value prefix. + uint32_t Suffix; // Offset (into strings) of value suffix. +} HMapBucket; + +typedef struct HMapHeader { + uint32_t Magic; // Magic word, also indicates byte order. + uint16_t Version; // Version number -- currently 1. + uint16_t Reserved; // Reserved for future use - zero for now. + uint32_t StringsOffset; // Offset to start of string pool. + uint32_t NumEntries; // Number of entries in the string table. + uint32_t NumBuckets; // Number of buckets (always a power of 2). + uint32_t MaxValueLength; // Length of longest result path (excluding nul). + // An array of 'NumBuckets' HMapBucket objects + // follows this header. Strings follow the + // buckets, at StringsOffset. +} HMapHeader; + +typedef struct HeaderMap { + char* data; // storage for the header map + size_t size; // size of data storage + int needsSwap; // endianness + uint32_t stringsTableNextEntry; // offset of where the next entry goes +} HeaderMap; + +#define HEADER(hmap) ((HMapHeader*)hmap->data) +#define BUCKET_TABLE(hmap) ((HMapBucket*)(hmap->data + sizeof(HMapHeader))) + +static HMapBucket getBucket(HeaderMap* hmap, unsigned int bucketNumber); +static HMapBucket* findBucketForKey(HeaderMap* hmap, char* key); +static HMapBucket* findEmptyBucket(HeaderMap* hmap, char* key); +static char* getStringFromTable(HeaderMap* hmap, unsigned int tblIndex); +static inline int isPowerOf2(uint32_t v); +static inline int nextPowerOf2(uint32_t v); +static inline unsigned addStringToTable(HeaderMap* hmap, char* str); +static inline unsigned hmapKey(char* str); +static int checkHeader(HeaderMap*); +static void dumpBuckets(HMapBucket* table, unsigned size); +static void dumpHeader(HMapHeader* header); +static void dumpStringsTable(char* start, unsigned num, char* end); + +// PUBLIC API + +HeaderMap* hmap_open(char* path, char* mode) { + int oflags = O_RDONLY; + int mprot = PROT_READ; + if (strstr(mode, "w")) { + oflags = O_RDWR; + mprot |= PROT_WRITE; + } + int fd = open(path, oflags, 0); + if (fd < 0) { + fprintf(stderr, "open(%s): %s\n", path, strerror(errno)); + return NULL; + } + struct stat st; + if (fstat(fd, &st) != 0) { + fprintf(stderr, "fstat(%s): %s\n", path, strerror(errno)); + close(fd); + return NULL; + } + char* data = mmap(0, st.st_size, mprot, MAP_SHARED, fd, 0); + if (data == MAP_FAILED) { + fprintf(stderr, "mmap(%s): %s\n", path, strerror(errno)); + close(fd); + return NULL; + } + // POSIX says it's safe to close the file descriptor + // http://pubs.opengroup.org/onlinepubs/7908799/xsh/mmap.html + close(fd); + HeaderMap* hmap = (HeaderMap*)calloc(1, sizeof(HeaderMap)); + hmap->data = data; + hmap->size = st.st_size; + hmap->stringsTableNextEntry = hmap->size + 1; + if (!checkHeader(hmap)) { + fprintf(stderr, "invalid hmap header\n"); + hmap_close(hmap); + return NULL; + } + return hmap; +} + +HeaderMap* hmap_new(unsigned numKeys) { + HeaderMap* hmap = (HeaderMap*)calloc(1, sizeof(HeaderMap)); + unsigned numBuckets = nextPowerOf2(numKeys * 3); // default is 1/3d full + unsigned stringOffset = + sizeof(HMapHeader) + numBuckets * sizeof(HMapBucket); + hmap->size = stringOffset + 1; + hmap->stringsTableNextEntry = stringOffset + 1; + hmap->data = calloc(1, hmap->size); + HMapHeader* header = (HMapHeader*)hmap->data; + // fill in all the boring fields + header->Magic = HMAP_HeaderMagicNumber; + header->Version = HMAP_HeaderVersion; + header->Reserved = 0; + header->MaxValueLength = 0; + header->NumBuckets = numBuckets; + header->NumEntries = 0; + header->StringsOffset = stringOffset; + return hmap; +} + +int hmap_addEntry(HeaderMap* hmap, char* key, char* value) { + char* prefix; + asprintf(&prefix, "%s/", dirname(value)); + char* suffix = basename(value); + + uint32_t key_idx = addStringToTable(hmap, key); + uint32_t prefix_idx = addStringToTable(hmap, prefix); + uint32_t suffix_idx = addStringToTable(hmap, suffix); + + HMapBucket* bucket = findEmptyBucket(hmap, key); + if (!bucket) { + // table full + return 1; + } + bucket->Key = key_idx; + bucket->Prefix = prefix_idx; + bucket->Suffix = suffix_idx; + + HEADER(hmap)->MaxValueLength = + max(HEADER(hmap)->MaxValueLength, strlen(value)); + HEADER(hmap)->NumEntries += 3; // key, prefix, suffix + free(prefix); + return 0; +} + +int hmap_findEntry(HeaderMap* hmap, char* key, char** value) { + HMapBucket* bucket = findBucketForKey(hmap, key); + if (!bucket) { + *value = NULL; + return 1; + } + char* prefix = getStringFromTable(hmap, bucket->Prefix); + char* suffix = getStringFromTable(hmap, bucket->Suffix); + asprintf(value, "%s%s", prefix, suffix); + return 0; +} + +void hmap_free(HeaderMap* hmap) { + free(hmap->data); + free(hmap); +} + +void hmap_close(HeaderMap* hmap) { + munmap(hmap->data, hmap->size); + free(hmap); +} + +int hmap_save(HeaderMap* hmap, char* path) { + int fd = open(path, O_CREAT | O_WRONLY, 0644); + if (fd < 0) { + perror(path); + return 1; + } + int rc = 0; + if (write(fd, hmap->data, hmap->stringsTableNextEntry) != + hmap->stringsTableNextEntry) { + perror(path); + rc = 1; + } + + // Don't fsync if it failed + if (rc != 1 && fsync(fd) < 0) { + perror(path); + rc = 1; + } + + // Close regardless of failure + if (close(fd)) { + perror(path); + rc = 1; + } + return rc; +} + +void hmap_dump(HeaderMap* hmap) { + printf("Header:\n"); + dumpHeader(HEADER(hmap)); + printf("Buckets:\n"); + dumpBuckets((HMapBucket*)(hmap->data + sizeof(HMapHeader)), + HEADER(hmap)->NumBuckets); + printf("StringsTable:\n"); + dumpStringsTable((hmap->data + HEADER(hmap)->StringsOffset), + HEADER(hmap)->NumEntries, hmap->data + hmap->size); + printf("Entries:\n"); + for (unsigned int i = 0; i < HEADER(hmap)->NumBuckets; ++i) { + HMapBucket b = getBucket(hmap, i); + if (b.Key == HMAP_EmptyBucketKey) continue; + char* key = getStringFromTable(hmap, b.Key); + char* prefix = getStringFromTable(hmap, b.Prefix); + char* suffix = getStringFromTable(hmap, b.Suffix); + printf("[%u] '%s' -> '%s' '%s'\n", i, key, prefix, suffix); + } +} + +// utility function helpful for iterating over a header map. Not +// intended to be called directly but through the macro HMAP_EACH() +int hmap_getidx(HeaderMap* hmap, unsigned start, char** key, char** val) { + if (start >= HEADER(hmap)->NumBuckets) return 0; + for (unsigned i = start; i < HEADER(hmap)->NumBuckets; ++i) { + HMapBucket b = getBucket(hmap, i); + if (b.Key == HMAP_EmptyBucketKey) continue; + *key = strdup(getStringFromTable(hmap, b.Key)); + asprintf(val, "%s%s", getStringFromTable(hmap, b.Prefix), + getStringFromTable(hmap, b.Suffix)); + return i + 1; + } + return 0; +} + +// Internal functions + +static inline unsigned hmapKey(char* str) { + unsigned result = 0; + char* end = str + strlen(str); + for (char* c = str; c < end; c++) result += tolower(*c) * 13; + return result; +} + +static inline int isPowerOf2(uint32_t v) { return v && !(v & (v - 1)); } +static inline int nextPowerOf2(uint32_t v) { + if (isPowerOf2(v)) return v; + unsigned n = 0; + while (v != 0) { + v >>= 1; + n++; + } + return 1 << n; +} + +static void dumpHeader(HMapHeader* header) { + uint32_t magic = header->Magic; + printf("Magic: %c%c%c%c\n", magic >> 24, magic >> 16 & 0xff, + magic >> 8 & 0xff, magic & 0xff); + printf("Version: %u\n", header->Version); + printf("Reserved: %u\n", header->Reserved); + printf("StringsOffset: %u\n", header->StringsOffset); + printf("NumEntries: %u\n", header->NumEntries); + printf("NumBuckets: %u\n", header->NumBuckets); + printf("MaxValueLength: %u\n", header->MaxValueLength); +} + +static void dumpBuckets(HMapBucket* table, unsigned size) { + for (unsigned i = 0; i < size; ++i) { + if (table[i].Key == HMAP_EmptyBucketKey) { + printf("%u: empty\n", i); + } else { + printf("%u: %u %u %u\n", i, table[i].Key, table[i].Prefix, + table[i].Suffix); + } + } +} + +static void dumpStringsTable(char* start, unsigned num, char* end) { + unsigned i = 0; + unsigned offset = 0; + while (i < num + 1) { + printf("%u: %s\n", offset, start); + unsigned len = strlen(start) + 1; + start += len; + offset += len; + + if (start > end) { + fprintf(stderr, "StringsTable Overrun!\n"); + break; + } + i++; + } +} + +static uint32_t maybe_bswap32(HeaderMap* hmap, uint32_t v) { + if (hmap->needsSwap) { + return bswap_32(v); + } + return v; +} + +// Checks that the header is a valid header map header. +static int checkHeader(HeaderMap* hmap) { + if (hmap->size <= sizeof(HMapHeader)) { + return 0; + } + HMapHeader* header = (HMapHeader*)hmap->data; + if (header->Magic == HMAP_HeaderMagicNumber && + header->Version == HMAP_HeaderVersion) { + hmap->needsSwap = 0; + } else if (header->Magic == bswap_32(HMAP_HeaderMagicNumber) && + header->Version == bswap_16(HMAP_HeaderVersion)) { + hmap->needsSwap = 1; + } else { + // not a header map + return 0; + } + // Not sure why this is, it was in llvm's implementation + if (header->Reserved != 0) { + return 0; + } + // check number of buckets, it should be a power of two and there + // should be enough space in the file for all of them. + uint32_t numBuckets = maybe_bswap32(hmap, header->NumBuckets); + if (!isPowerOf2(numBuckets)) { + return 0; + } + if (hmap->size < sizeof(HMapHeader) + sizeof(HMapBucket) * numBuckets) { + return 0; + } + return 1; +} + +// Get bucket from HeaderMap data structure +static HMapBucket getBucket(HeaderMap* hmap, unsigned int bucketNumber) { + assert(hmap); + assert(hmap->size >= + sizeof(HMapHeader) + sizeof(HMapBucket) * bucketNumber); + HMapBucket result; + result.Key = HMAP_EmptyBucketKey; + HMapBucket* bucketArray = BUCKET_TABLE(hmap); + HMapBucket* bucket = bucketArray + bucketNumber; + // load values, byte-swapping as needed + result.Key = maybe_bswap32(hmap, bucket->Key); + result.Prefix = maybe_bswap32(hmap, bucket->Prefix); + result.Suffix = maybe_bswap32(hmap, bucket->Suffix); + return result; +} + +static HMapBucket* findEmptyBucket(HeaderMap* hmap, char* key) { + unsigned hkey = hmapKey(key); + HMapBucket* table = BUCKET_TABLE(hmap); + uint32_t numBuckets = HEADER(hmap)->NumBuckets; + assert(numBuckets); + unsigned i; + for (i = 0; i < numBuckets; ++i) { + unsigned idx = (hkey + i) % numBuckets; + if (table[idx].Key == HMAP_EmptyBucketKey) { + return &(table[idx]); + } + } + // we couldn't find an empty bucket + return NULL; +} + +static HMapBucket* findBucketForKey(HeaderMap* hmap, char* key) { + unsigned hkey = hmapKey(key); + HMapBucket* table = BUCKET_TABLE(hmap); + uint32_t numBuckets = HEADER(hmap)->NumBuckets; + unsigned i; + for (i = 0; i < numBuckets; ++i) { + unsigned idx = (hkey + i) % numBuckets; + if (table[idx].Key == HMAP_EmptyBucketKey) { + // didn't find it + return NULL; + } + char* bucketKey = getStringFromTable(hmap, table[idx].Key); + if (!bucketKey) { + continue; + } + if (strcmp(bucketKey, key)) { + continue; + } + // match + return &(table[idx]); + } + return NULL; +} + +static char* getStringFromTable(HeaderMap* hmap, unsigned int tblIndex) { + tblIndex += HEADER(hmap)->StringsOffset; + if (tblIndex >= hmap->size) { + return NULL; + } + char* data = hmap->data + tblIndex; + unsigned maxLen = hmap->size - tblIndex; + unsigned len = strnlen(data, maxLen); + // make sure it's null terminated + if (len == maxLen && data[len]) { + fprintf(stderr, "Invalid string at %u. Not NULL terminated\n", + tblIndex); + assert(0); + } + return data; +} + +static inline unsigned addStringToTable(HeaderMap* hmap, char* str) { + unsigned idx = hmap->stringsTableNextEntry; + unsigned len = strlen(str); + if (idx + len + 1 >= hmap->size) { + while (idx + len + 1 >= hmap->size) { hmap->size *= 2; } + hmap->data = reallocf(hmap->data, hmap->size); + assert(hmap->data); + } + memcpy(hmap->data + idx, str, len + 1); + hmap->stringsTableNextEntry += len + 1; + hmap->data[hmap->stringsTableNextEntry] = '\0'; + + return idx - HEADER(hmap)->StringsOffset; +} diff --git a/tools/hmaptool/hmap.h b/tools/hmaptool/hmap.h new file mode 100644 index 0000000000..4751e27c8e --- /dev/null +++ b/tools/hmaptool/hmap.h @@ -0,0 +1,23 @@ +#ifndef _HMAP_H +#define _HMAP_H + +typedef struct HeaderMap HeaderMap; + +#define HMAP_EACH(hmap, key, value) \ + for (unsigned __idx = 0; \ + (__idx = hmap_getidx(hmap, __idx, &(key), &(value)));) + +HeaderMap* hmap_open(char* path, char* mode); +void hmap_close(HeaderMap*); + +HeaderMap* hmap_new(unsigned size); +void hmap_free(HeaderMap*); + +int hmap_save(HeaderMap* hmap, char* path); + +int hmap_addEntry(HeaderMap* hmap, char* key, char* value); +int hmap_findEntry(HeaderMap* hmap, char* key, char** value); +int hmap_getidx(HeaderMap* hmap, unsigned start, char** key, char** val); +void hmap_dump(HeaderMap* hmap); + +#endif /* _HMAP_H */ diff --git a/tools/hmaptool/hmaptool.c b/tools/hmaptool/hmaptool.c new file mode 100644 index 0000000000..5ceb688d55 --- /dev/null +++ b/tools/hmaptool/hmaptool.c @@ -0,0 +1,188 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hmap.h" +#include "lines.h" +#include "uthash.h" + +static int verbose = 0; + +typedef struct mapping { + char *key; + char *value; + UT_hash_handle hh; +} mapping; + +static struct { + char *name_space; + char *output_file; +} cli_args; + +static void usage(); +static void debug(char *format, ...); + +static inline void chomp(char *s); +static void add_entry(mapping **hashmap, char *key, char *value); +static void add_header(mapping **hashmap, char *name_space, char *header); +static void parse_args(mapping **hashmap, char **av, int ac); +static void parse_param_file(mapping **hashmap, char *file); + +static struct option longopts[] = { + {"namespace", required_argument, NULL, 'n'}, + {"output", required_argument, NULL, 'o'}, + {"verbose", no_argument, NULL, 'v'}, + {NULL, 0, NULL, 0}, +}; + +int main(int ac, char **av) { + cli_args.name_space = NULL; + cli_args.output_file = NULL; + mapping *entries = NULL; + + parse_args(&entries, av, ac); + + mapping *m; + unsigned numEntries = 0; + for (m = entries; m != NULL; m = m->hh.next) { + numEntries++; + } + debug("Writing hmap with %u entries", numEntries); + HeaderMap *hmap = hmap_new(numEntries); + for (m = entries; m != NULL; m = m->hh.next) { + if (hmap_addEntry(hmap, m->key, m->value)) { + fprintf(stderr, "failed to add '%s' to hmap\n", m->key); + } + } + int rc = 0; + rc |= hmap_save(hmap, cli_args.output_file); + hmap_free(hmap); + return rc; +} + +static void usage() { + fprintf(stderr, "hmaptool: "); + for (struct option *op = longopts; op->name; ++op) { + fprintf(stderr, "[--%s", op->name); + if (op->has_arg == required_argument) fprintf(stderr, " "); + fprintf(stderr, "]"); + } + fprintf(stderr, " [...]\n"); + exit(1); +} + +static void debug(char *format, ...) { + if (!verbose) return; + char *buffer = NULL; + va_list args; + va_start(args, format); + vasprintf(&buffer, format, args); + printf("%s\n", buffer); + free(buffer); + va_end(args); +} + +static void parse_args(mapping **entries, char **av, int ac) { + int c; + optind = 0; + while ((c = getopt_long(ac, av, "n:o:", longopts, NULL)) != -1) { + switch (c) { + case 'n': + cli_args.name_space = strdup(optarg); + break; + case 'o': + cli_args.output_file = strdup(optarg); + break; + case 'v': + verbose = 1; + break; + default: + usage(); + } + } + ac -= optind; + av += optind; + + // all remaining arguments are the actual headers + for (; *av; av++) { + if (**av == '@') { + // param file + parse_param_file(entries, *av); + continue; + } + add_header(entries, cli_args.name_space, *av); + } +} + +static void add_header(mapping **hashmap, char *name_space, char *header) { + char *bn = strdup(basename(header)); + if (bn == NULL) { + fprintf(stderr, + "Failed to parse '%s': could not extract basename: %s\n", + header, strerror(errno)); + exit(1); + } + add_entry(hashmap, bn, strdup(header)); + if (name_space) { + char *key = NULL; + asprintf(&key, "%s/%s", name_space, bn); + if (!key) { + perror("malloc"); + exit(1); + } + add_entry(hashmap, key, strdup(header)); + } +} + +static void add_entry(mapping **hashmap, char *key, char *value) { + mapping *entry = NULL; + HASH_FIND_STR(*hashmap, key, entry); + if (entry) { + // key already in the hash, verify the value + if (strcmp(value, entry->value)) { + fprintf(stderr, + "WARNING: header '%s' already in cache as '%s' ignoring " + "new value: '%s'\n", + key, entry->value, value); + } + return; + } + entry = calloc(1, sizeof(mapping)); + entry->key = key; + entry->value = value; + HASH_ADD_KEYPTR(hh, *hashmap, entry->key, strlen(entry->key), entry); +} + +static inline void chomp(char *s) { + unsigned len = strlen(s); + if (s[len - 1] == '\n') s[len - 1] = '\0'; +} + +static void parse_param_file(mapping **hashmap, char *file) { + assert(file[0] == '@'); + file++; // skip @ + + FILE *f = fopen(file, "r"); + if (!f) { + perror(file); + exit(1); + } + ssize_t nread; + char *line = NULL; + size_t len; + char **av = NULL; + while ((nread = getline(&line, &len, f)) != -1) { + chomp(line); + av = addLine(av, line); + line = NULL; + } + av = addLine(av, NULL); + fclose(f); + parse_args(hashmap, av, nLines(av)); +} diff --git a/tools/hmaptool/lines.c b/tools/hmaptool/lines.c new file mode 100644 index 0000000000..a49438d7be --- /dev/null +++ b/tools/hmaptool/lines.c @@ -0,0 +1,716 @@ +/* + * Copyright 1997-2013,2015-2016 BitMover, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * Note: This file is also used by src/win32/pub/diffutils + * Do not put BitKeeper specific code here + */ + +#include "lines.h" + +#include +#include +#include +#include +#include +#include +#include + +typedef uint8_t u8; +typedef uint32_t u32; + +#define streq(a, b) (!strcmp((a), (b))) +#define unless(e) if (!(e)) +#define INVALID (void *)~0u /* invalid pointer */ + +#define setLLEN(s, len) (*(u32 *)(s) = (*(u32 *)(s) & ~LMASK) | len) + +/* size of array (saves LSIZ-1 items) */ +#define LSIZ(s) (1u << (*(u32 *)(s) >> LBITS)) + +/* + * add 'add' more elements to an array, resizing as needed + * internal version that doesn't clear memory + */ +static void *_growArray_int(void *space, int add, int elemsize) { + u32 size, len; /* len and size of array */ + int c; + + if (space) { + len = _LLEN(space) + add; + c = (*(u32 *)space >> LBITS); + size = 1u << c; + + assert(size > 0); /* size==0 is read-only array */ + } else { + assert(elemsize >= sizeof(u32)); + len = add; + c = (elemsize > 128) ? 2 : 3; /* min alloc size */ + size = 1u << c; + goto alloc; + } + if (len >= size) { /* full up, dude */ + alloc: + while (len >= size) { + size *= 2; + ++c; + } + space = realloc(space, size * elemsize); + assert(space); + } + *(u32 *)space = (c << LBITS) | len; + return (space); +} + +/* + * add 'add' more elements to an array, resizing as needed + * new elements are set to zero + * returns: a pointer to the new elements + */ +void *_growArray(void **space, int add, int size) { + void *ret; + + *space = _growArray_int(*space, add, size); + ret = *(u8 **)space + (_LLEN(*space) - add + 1) * size; + memset(ret, 0, add * size); + return (ret); +} + +/* + * Return a new array of length 0 with space for n lines. + */ +char **allocLines(int n) { + char **space = _growArray_int(0, n, sizeof(char *)); + + setLLEN(space, 0); + return (space); +} + +/* + * Add a char* to the end of an array, if line==0 then the array is + * zero terminated, but the length doesn't actually change. + */ +char **addLine(char **space, void *line) { + int len; + + space = _growArray_int(space, 1, sizeof(char *)); + len = nLines(space); + space[len] = line; + unless(line) setLLEN(space, len - 1); + return (space); +} + +int findLine(char **haystack, char *needle) { + int i; + + EACH(haystack) { + if (streq(needle, haystack[i])) return (i); + } + return (0); +} + +/* + * Adds 1 new element to array and copies 'x' into, if + * x==0 then the new item is cleared. + * returns a pointer to the new item. + */ +void *_addArray(void **space, void *x, int size) { + void *ret; + int len; + + *space = _growArray_int(*space, 1, size); + len = nLines(*space); + ret = (u8 *)(*space) + len * size; + if (x) { + memcpy(ret, x, size); + } else { + memset(ret, 0, size); + } + return (ret); +} + +/* + * set the length of a lines array to a new (smaller) value. + */ +void truncArray(void *space, int len) { + if (space) { + /* no growing the array at the moment */ + assert(len <= _LLEN(space)); + setLLEN(space, len); + } else { + assert(len == 0); + } +} + +/* + * copy array to the end of space and then zero array + */ +void *_catArray(void **space, void *array, int size) { + int n1, n2; + void *ret = 0; + + if (n2 = nLines(array)) { + n1 = nLines(*space); + *space = _growArray_int(*space, n2, size); + ret = (u8 *)*space + (n1 + 1) * size; + memcpy(ret, (u8 *)array + size, n2 * size); + setLLEN(array, 0); + } + return (ret); +} + +void reverseLines(char **space) { + int i, end; + char *tmp; + + end = nLines(space); + i = 1; + while (i < end) { + tmp = space[i]; + space[i] = space[end]; + space[end] = tmp; + ++i; + --end; + } +} + +void _reverseArray(void *space, int size) { + int i, end; + u8 tmp[size], *arr = (u8 *)space; + + end = nLines(space); + i = 1; + while (i < end) { + memcpy(&tmp, &arr[i * size], size); + memcpy(&arr[i * size], &arr[end * size], size); + memcpy(&arr[end * size], &tmp, size); + ++i; + --end; + } +} + +void _sortArray(void *space, int (*compar)(const void *, const void *), + int size) { + if (!space) return; + unless(compar) compar = string_sort; + qsort((u8 *)space + size, nLines(space), size, compar); +} + +/* + * Looking for a key_sort? It's in check.c and sorts on the key starting + * at the pathname part. + */ + +int string_sort(const void *a, const void *b) { + char *l, *r; + + l = *(char **)a; + r = *(char **)b; + return (strcmp(l, r)); +} + +int stringcase_sort(const void *a, const void *b) { + char *l, *r; + + l = *(char **)a; + r = *(char **)b; + return (strcasecmp(l, r)); +} + +int string_sortrev(const void *a, const void *b) { + char *l, *r; + + l = *(char **)a; + r = *(char **)b; + return (strcmp(r, l)); /* reverse sort */ +} + +int number_sort(const void *a, const void *b) { + int l, r; + + l = atoi(*(char **)a); + r = atoi(*(char **)b); + if (l - r) return (l - r); + return (string_sort(a, b)); +} + +void uniqLines(char **space, void (*freep)(void *ptr)) { + int src, dst; + + unless(space) return; + sortLines(space, 0); + + /* skip up to the first dup */ + EACH_INDEX(space, src) { + if ((src > 1) && streq(space[src - 1], space[src])) goto dups; + } + return; /* fast exit */ + +dups: /* now copy non-duped items */ + dst = src - 1; /* last valid item in output */ + for (; src <= _LLEN(space); src++) { /* EACH() */ + if (streq(space[src], space[dst])) { + if (freep) freep(space[src]); + } else { + space[++dst] = space[src]; + } + } + truncLines(space, dst); +} + +/* + * Return true if they are the same. + * It's up to you to sort them first if you want them sorted. + */ +int sameLines(char **p, char **p2) { + int i; + + unless(p && p2) return (0); + unless(nLines(p) == nLines(p2)) return (0); + EACH(p) unless(streq(p[i], p2[i])) return (0); + return (1); +} + +void freeLines(char **space, void (*freep)(void *ptr)) { + int i; + + if (!space || (space == INVALID)) return; + if (freep) { + EACH(space) freep(space[i]); + } + space[0] = 0; + free(space); +} + +/* same non O(n^2) idiom as uniqLines() */ +int removeLine(char **space, char *s, void (*freep)(void *ptr)) { + int src, dst, n = 0; + + /* skip up to the first match */ + EACH_INDEX(space, src) { + if (streq(space[src], s)) goto match; + } + return (0); /* fast exit */ + +match: /* now copy non-matched items */ + dst = src - 1; /* last non-matched item in output */ + for (; src <= _LLEN(space); src++) { /* EACH() */ + if (streq(space[src], s)) { + n++; + if (freep) freep(space[src]); + } else { + space[++dst] = space[src]; + } + } + truncLines(space, dst); + return (n); +} + +/* + * set space[j] = line + * and shift everything down to make room + * return ptr to new item + */ +void *_insertArrayN(void **space, int j, void *new, int size) { + int len; + void *ret; + + len = nLines(*space) + 1; + assert((j > 0) && (j <= len)); + /* alloc spot and inc line count */ + *space = _growArray_int(*space, 1, size); + ret = (u8 *)*space + j * size; + if (j < len) memmove((u8 *)ret + size, ret, (len - j) * size); + if (new) { + memcpy(ret, new, size); + } else { + memset(ret, 0, size); + } + return (ret); +} + +void _removeArrayN(void *space, int rm, int size) { + int len = _LLEN(space); + + assert(rm <= len); + assert(rm > 0); + if (rm < len) { + memmove((u8 *)space + rm * size, (u8 *)space + (rm + 1) * size, + (len - rm) * size); + } + setLLEN(space, len - 1); +} + +/* + * A simple wrapper for removeArray() to use for an array of pointers. + * If freep is passed, then we free the pointer being removed + * and return zero. Otherwise we return the pointer removed + * from the array. + */ +void *removeLineN(char **space, int rm, void (*freep)(void *ptr)) { + char *ret; + + if (freep) { + freep(space[rm]); + ret = 0; + } else { + if ((rm < 1) || (rm > nLines(space))) return (0); + ret = space[rm]; + } + _removeArrayN(space, rm, sizeof(void *)); + return (ret); +} + +/* + * Like perl's join(), + * use it for making arbitrary length strings. + */ +char *joinLines(char *sep, char **space) { + int i, slen, len = 0; + char *buf, *p; + + if (emptyLines(space)) return (0); + slen = sep ? strlen(sep) : 0; + EACH(space) { + len += strlen(space[i]); + len += slen; + } + len++; + buf = malloc(len); + p = buf; + EACH(space) { + strcpy(p, space[i]); + p += strlen(space[i]); + if (sep) { + strcpy(p, sep); + p += slen; + } + } + /* + * No trailing sep. + */ + if (sep) { + p -= slen; + *p = 0; + } + return (buf); +} + +/* + * Split a C string into tokens like strtok() + * + * The string 'line' is seperated into tokens seperated + * by one of more characters from 'delim' and each token will + * be added to the 'tokens' line array. + * The tokens will be null terminated and will not contain characters + * from 'delim' + */ +char **splitLine(char *line, char *delim, char **tokens) { + int len; + + while (1) { + line += strspn(line, delim); /* skip delimiters */ + len = strcspn(line, delim); + unless(len) break; + tokens = addLine(tokens, strndup(line, len)); + line += len; + } + return (tokens); +} + +/* + * Return a malloc'ed string with quotes such that it will be parsed + * as one argument by the shell. If a list of strings quoted by this + * function are joined by spaces and passed to shellSplit() above, the + * original strings will be returned. + * + * All characters in the input string are considered literals. + */ +char *shellquote(char *in) { + int len = strlen(in); + int nlen; + char *s, *t; + char *out; + + /* handle simple case */ + if (strcspn(in, " \t\n\r'\"<>|$&;[]*()\\") == len) return (strdup(in)); + + nlen = len + 2 + 1; /* quotes + null */ + for (s = in; *s; s++) { + if ((*s == '"') || (*s == '\\')) ++nlen; + } + t = out = malloc(nlen); + *t++ = '"'; + for (s = in; *s;) { + if ((*s == '"') || (*s == '\\')) *t++ = '\\'; + *t++ = *s++; + } + *t++ = '"'; + *t = 0; + return (out); +} + +/* + * Takes a string, parses it like /bin/sh, and splits it into + * tokens. The result is is returned in a lines array. + * + * Rules: + * + * The string is split on white space boundaries unless the white space + * is made literal by one of the following rules. + * + * A backslash (\) is the escape character. It preservers the + * literal value of the next character that follows. + * + * Enclosing characters in single quotes preserves the literal + * value of each character within the quotes. A single quote may + * not occur between single quotes, even when preceded by a + * backslash. + * + * Enclosing characters in double quotes preserves the literal + * value of all characters within the quotes, with the exception + * of \. The backslash retains its special meaning only when + * followed by the characters " or \. A double quote (") may be + * quoted within double quotes by preceding it with a backslash. + * + * As a special flag for spawn_pipeline() every string has a 1 + * character flag after the null at the end. If it is 1 then that + * item is a shell meta function (like > or |). It is 0 otherwise. + * This is so '>' and \> can be seperated from >. + * (the marker is not visible unless you know to look for it.) + */ +char **shellSplit(const char *line) { + const char *pin; /* input pointer */ + const char *ein; /* pointer to end of block to insert */ + char **ret = 0; + char *e; + int item = 0; /* buf contains data */ + int c; + int bufsize = 128; + char *buf = malloc(bufsize); + char *pout; /* end of 'buf' at insert point */ + +/* make room for 'extra' more characters in buffer */ +#define BUF_RESIZE(extra) \ + while (pout - buf > bufsize - (extra)) { \ + char *newbuf = malloc(2 * bufsize); \ + memcpy(newbuf, buf, pout - buf); \ + pout = newbuf + (pout - buf); \ + buf = newbuf; \ + bufsize *= 2; \ + } + +/* If we have an item, save it one the list and reset the buffer */ +#define SAVE_ITEM() \ + if (item) { \ + *pout++ = 0; /* 2 nulls */ \ + *pout++ = 0; \ + ret = addLine(ret, memcpy(malloc(pout - buf), buf, pout - buf)); \ + pout = buf; \ + item = 0; \ + } + + assert(line); + pin = line; + pout = buf; + *pout = 0; + while (*pin) { + switch (*pin) { + /* whitespace */ + case ' ': + case '\t': + case '\n': + case '\r': + ++pin; + SAVE_ITEM(); + break; + /* single quoted strings */ + case '\'': + ++pin; + ein = strchr(pin, '\''); + unless(ein) { + fprintf(stderr, "unmatched ' in (%s)\n", line); + exit(1); + } + BUF_RESIZE(ein - pin); + strncpy(pout, pin, ein - pin); + pout += ein - pin; + pin = ein + 1; + item = 1; + break; + /* double quoted strings */ + case '"': + ++pin; + while (*pin && *pin != '"') { + ein = pin + strcspn(pin, "\"\\"); + BUF_RESIZE(ein - pin + 1); + strncpy(pout, pin, ein - pin); + pout += ein - pin; + pin = ein; + if (*pin == '\\') { + if (pin[1] == '\\' || pin[1] == '"') { + ++pin; + } + *pout++ = *pin++; + } + } + if (*pin != '"') { + fprintf(stderr, "unmatched \" in (%s)\n", line); + exit(1); + } + ++pin; + item = 1; + break; + /* shell characters */ + case '>': + case '<': + case '|': + c = 1; + if (pin[0] == '|' && pin[1] == '|') goto unhandled; + if (pin[0] == '>' && pin[1] == '>') c = 2; + sep: + SAVE_ITEM(); + e = malloc(c + 2); + strncpy(e, pin, c); + e[c] = 0; + e[c + 1] = 1; /* 1 after null */ + ret = addLine(ret, e); + pin += c; + break; + /* unhandled functions */ + case '`': + case '$': + case '&': + case ';': + case '[': + case ']': + case '*': + case '(': + case ')': + unhandled: + fprintf( + stderr, + "command line (%s) contains unhandled shell expressions.\n", + line); + exit(1); + /* escape character */ + case '\\': + ++pin; + unless(*pin) break; + /* falltrough */ + default: + if (isdigit(*pin) && (pin[1] == '<' || pin[1] == '>')) { + if (pin[2] == '&' && isdigit(pin[3])) { + c = 4; + } else { + c = 2; + } + goto sep; + } else { + *pout++ = *pin++; + item = 1; + } + break; + } + BUF_RESIZE(2); + } + SAVE_ITEM(); + free(buf); + return (ret); +} + +/* + * Return an array allocated by different means. + * While it has no data, it will look like it does (nLines == len). + * The caller is expected to fill before using as an array. + */ +void *allocArray(int len, int esize, void *(*allocate)(size_t size)) { + int c; /* log2 max num elements */ + size_t size; + void *ret; + + assert(esize >= sizeof(u32)); + unless(allocate) allocate = malloc; + c = 4; + size = 1u << c; + while (len >= size) { + size *= 2; + ++c; + } + size *= esize; + ret = allocate(size); + *(u32 *)ret = (c << LBITS) | len; + return (ret); +} + +/* + * Given two arrays in sorted order, walk the arrays in parallel and + * call the supplied 'walk' call back on each it item found in either + * array. The walk callback is called with a pointer to the data in + * each array, if an item appear only in on array then the point for + * the other array will be 0. + * + * example: + * a = { "a", "b", "c" } + * b = { "b", "d" }; + * will make the following callbacks: + * walk(t, "a", 0); + * walk(t, "b", "b"); + * walk(t, "c", 0); + * walk(t, "0, "d"); + * + * 'compar' is the comparison function is set to string_sort if null + * If walk() returns a positive number then that number is accumulated + * and returned from parallelLines(). If the return from walk() is + * negative the tranversal is aborted and that number is returned + * immediately. + */ +int parallelLines(char **a, char **b, int (*compar)(const void *, const void *), + int (*walk)(void *token, char *a, char *b), void *token) { + int i, j; + int cmp, r; + int sum = 0; + int na, nb; + char *aa, *bb; + + unless(compar) compar = string_sort; + na = nLines(a); + nb = nLines(b); + + i = j = 1; + while ((i <= na) || (j <= nb)) { + if (i > na) { + cmp = 1; + } else if (j > nb) { + cmp = -1; + } else { + cmp = compar(a + i, b + j); + } + if (cmp < 0) { + aa = a[i++]; + bb = 0; + } else if (cmp > 0) { + aa = 0; + bb = b[j++]; + } else { + aa = a[i++]; + bb = b[j++]; + } + if ((r = walk(token, aa, bb)) < 0) return (r); + sum += r; + } + return (sum); +} diff --git a/tools/hmaptool/lines.h b/tools/hmaptool/lines.h new file mode 100644 index 0000000000..9dedf33c5e --- /dev/null +++ b/tools/hmaptool/lines.h @@ -0,0 +1,211 @@ +/* + * Copyright 2002-2007,2009-2016 BitMover, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * liblines - interfaces for autoexpanding data structures + * + * s= allocLines(n) + * pre allocate space for slightly less than N entries. + * s = addLine(s, line) + * add line to s, allocating as needed. + * line must be a pointer to preallocated space. + * freeLines(s, freep) + * free the lines array; if freep is set, call that on each entry. + * if freep is 0, do not free each entry. + * buf = popLine(s) + * return the most recently added line (not an alloced copy of it) + * reverseLines(s) + * reverse the order of the lines in the array + * sortLines(space, compar) + * sort the lines using the compar function if set, else string_sort() + * removeLine(s, which, freep) + * look for all lines which match "which" and remove them from the array + * returns number of matches found + * removeLineN(s, i, freep) + * remove the 'i'th line. + * lines = splitLine(buf, delim, lines) + * split buf on any/all chars in delim and put the tokens in lines. + * buf = joinLines(":", s) + * return one string which is all the strings glued together with ":" + * does not free s, caller must free s. + * buf = findLine(lines, needle); + * Return the index the line in lines that matches needle + */ +#ifndef _LIB_LINES_H +#define _LIB_LINES_H + +#include +#include + +typedef uint32_t u32; + +/* lines are limited to 2^27 entries ~134 million */ +#define LBITS (32 - 5) +#define LMASK 0x07ffffff + +/* sub-macro for EACH */ +/* length of array (use nLines() in code) */ +#define _LLEN(s) (*(u32 *)(s)&LMASK) + +#define L(d) \ + ({ \ + typeof(d) _d = d; \ + (_d) ? (typeof(_d)[]){(typeof(_d))1, (typeof(_d))_d} : 0; \ + }) + +#define EACH_START(x, s, i) \ + if ((i = (x)), (s)) \ + for (; (i) <= _LLEN(s); i++) +#define EACH_INDEX(s, i) EACH_START(1, s, i) +#define EACH(s) EACH_START(1, s, i) +#define EACH_REVERSE_INDEX(s, i) for (i = nLines(s); i > 0; i--) +#define EACH_REVERSE(s) EACH_REVERSE_INDEX(s, i) + +/* EACHP like EACH but walks a pointer to each array element instead of i */ +#define EACHP(s, ptr) \ + if (s) \ + for ((ptr) = (s) + 1; (ptr) <= (s) + _LLEN(s); ++(ptr)) +#define EACHP_REVERSE(s, ptr) \ + if (s) \ + for ((ptr) = (s) + _LLEN(s); (ptr) > (s); --(ptr)) + +#define EACH_STRUCT(s, c, i) \ + if (((c) = 0), (i = 1), (s)) \ + for (((c) = (void *)(s)[i]); \ + ((i) <= _LLEN(s)) && ((c) = (void *)(s)[i]); ++(i)) + +#define LNEXT(s) ((s) && ((i) <= _LLEN(s)) ? (s)[i++] : 0) + +/* return number of lines in array */ +#define nLines(s) ((s) ? _LLEN(s) : 0) +#define emptyLines(s) (nLines(s) == 0) + +char **addLine(char **space, void *line); +char **allocLines(int n); +#define truncLines(s, len) truncArray(s, len) +#define insertLineN(space, n, val) \ + ({ \ + void *str = (val); \ + char **_a = (space); \ + _insertArrayN((void **)&_a, n, &str, sizeof(char *)); \ + _a; \ + }) +void *removeLineN(char **space, int rm, void (*freep)(void *ptr)); +#define catLines(space, array) \ + ({ \ + char **_a = (space); \ + _catArray((void **)&_a, (array), sizeof(char *)); \ + _a; \ + }) + +#define pushLine(s, l) addLine(s, l) +#define popLine(s) removeLineN(s, nLines(s), 0) +#define shiftLine(s) removeLineN(s, 1, 0) +#define unshiftLine(s, val) insertLineN(s, 1, val) + +char **splitLine(char *line, char *delim, char **tokens); +char *joinLines(char *sep, char **space); +void freeLines(char **space, void (*freep)(void *ptr)); +int removeLine(char **space, char *s, void (*freep)(void *ptr)); +void reverseLines(char **space); +#define sortLines(s, compar) _sortArray(s, compar, sizeof(char *)) +int string_sort(const void *a, const void *b); +int stringcase_sort(const void *a, const void *b); +int string_sortrev(const void *a, const void *b); +int number_sort(const void *a, const void *b); +char **shellSplit(const char *line); +void uniqLines(char **space, void (*freep)(void *ptr)); +int sameLines(char **p, char **p2); +char *shellquote(char *in); +int findLine(char **haystack, char *needle); + +int parallelLines(char **a, char **b, int (*compar)(const void *, const void *), + int (*walk)(void *token, char *a, char *b), void *token); + +/* arrays of arbitrary sized data */ + +/* TYPE *growArray(TYPE **space, int n) */ +#define growArray(space, n) \ + ({ \ + typeof(*space) _ret; \ + _ret = _growArray((void **)space, n, sizeof(*_ret)); \ + _ret; \ + }) + +/* TYPE *addArray(TYPE **space, TYPE *new) */ +#define addArray(space, x) \ + ({ \ + typeof(*space) _arg = (x), _ret; \ + _ret = _addArray((void **)space, _arg, sizeof(*_ret)); \ + _ret; \ + }) + +/* void addArrayV(TYPE **space, TYPE new) */ +#define addArrayV(space, x) \ + (void)({ \ + typeof(**space) _arg = (x); \ + _addArray((void **)space, &_arg, sizeof(_arg)); \ + }) + +/* TYPE *insertArrayN(TYPE **space, int n, TYPE *x) */ +#define insertArrayN(space, n, x) \ + ({ \ + typeof(*space) _arg = (x), _ret; \ + _ret = _insertArrayN((void **)space, n, _arg, sizeof(*_ret)); \ + _ret; \ + }) + +/* void removeArrayN(TYPE *space, int n); */ +#define removeArrayN(s, n) _removeArrayN((s), (n), sizeof((s)[0])) + +/* void catArray(TYPE **space, TYPE *array) */ +#define catArray(s, a) _catArray((void **)(s), (a), sizeof((*(s))[0])) + +/* void reverseArray(TYPE *space); */ +#define reverseArray(s) _reverseArray((s), sizeof((s)[0])) + +/* void sortArray(TYPE *space, cmpfn); */ +#define sortArray(s, compar) _sortArray((s), (compar), sizeof((s)[0])) + +/* TYPE popArray(TYPE *space); */ +#define popArray(space) \ + ({ \ + int i = nLines(space); \ + typeof(*space) _ret; \ + if (i) { \ + _ret = space[i]; \ + removeArrayN(space, i); \ + } else { \ + memset(&_ret, 0, sizeof(_ret)); \ + } \ + _ret; \ + }) + +void *_growArray(void **space, int add, int size); +void *_addArray(void **space, void *x, int size); +void truncArray(void *space, int len); +void *_insertArrayN(void **space, int j, void *line, int size); +void _removeArrayN(void *space, int rm, int size); +void *_catArray(void **space, void *array, int size); +void _reverseArray(void *space, int size); +void _sortArray(void *space, int (*compar)(const void *, const void *), + int size); + +void *allocArray(int cnt, int size, void *(*allocate)(size_t len)); + +void lines_tests(void); + +#endif diff --git a/tools/hmaptool/uthash.h b/tools/hmaptool/uthash.h new file mode 100644 index 0000000000..a790ea59fa --- /dev/null +++ b/tools/hmaptool/uthash.h @@ -0,0 +1,1230 @@ +/* +Copyright (c) 2003-2018, Troy D. Hanson http://troydhanson.github.com/uthash/ +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef UTHASH_H +#define UTHASH_H + +#define UTHASH_VERSION 2.1.0 + +#include /* memcmp, memset, strlen */ +#include /* ptrdiff_t */ +#include /* exit */ + +/* These macros use decltype or the earlier __typeof GNU extension. + As decltype is only available in newer compilers (VS2010 or gcc 4.3+ + when compiling c++ source) this code uses whatever method is needed + or, for VS2008 where neither is available, uses casting workarounds. */ +#if !defined(DECLTYPE) && !defined(NO_DECLTYPE) +#if defined(_MSC_VER) /* MS compiler */ +#if _MSC_VER >= 1600 && defined(__cplusplus) /* VS2010 or newer in C++ mode */ +#define DECLTYPE(x) (decltype(x)) +#else /* VS2008 or older (or VS2010 in C mode) */ +#define NO_DECLTYPE +#endif +#elif defined(__BORLANDC__) || defined(__ICCARM__) || defined(__LCC__) || defined(__WATCOMC__) +#define NO_DECLTYPE +#else /* GNU, Sun and other compilers */ +#define DECLTYPE(x) (__typeof(x)) +#endif +#endif + +#ifdef NO_DECLTYPE +#define DECLTYPE(x) +#define DECLTYPE_ASSIGN(dst,src) \ +do { \ + char **_da_dst = (char**)(&(dst)); \ + *_da_dst = (char*)(src); \ +} while (0) +#else +#define DECLTYPE_ASSIGN(dst,src) \ +do { \ + (dst) = DECLTYPE(dst)(src); \ +} while (0) +#endif + +/* a number of the hash function use uint32_t which isn't defined on Pre VS2010 */ +#if defined(_WIN32) +#if defined(_MSC_VER) && _MSC_VER >= 1600 +#include +#elif defined(__WATCOMC__) || defined(__MINGW32__) || defined(__CYGWIN__) +#include +#else +typedef unsigned int uint32_t; +typedef unsigned char uint8_t; +#endif +#elif defined(__GNUC__) && !defined(__VXWORKS__) +#include +#else +typedef unsigned int uint32_t; +typedef unsigned char uint8_t; +#endif + +#ifndef uthash_malloc +#define uthash_malloc(sz) malloc(sz) /* malloc fcn */ +#endif +#ifndef uthash_free +#define uthash_free(ptr,sz) free(ptr) /* free fcn */ +#endif +#ifndef uthash_bzero +#define uthash_bzero(a,n) memset(a,'\0',n) +#endif +#ifndef uthash_strlen +#define uthash_strlen(s) strlen(s) +#endif + +#ifdef uthash_memcmp +/* This warning will not catch programs that define uthash_memcmp AFTER including uthash.h. */ +#warning "uthash_memcmp is deprecated; please use HASH_KEYCMP instead" +#else +#define uthash_memcmp(a,b,n) memcmp(a,b,n) +#endif + +#ifndef HASH_KEYCMP +#define HASH_KEYCMP(a,b,n) uthash_memcmp(a,b,n) +#endif + +#ifndef uthash_noexpand_fyi +#define uthash_noexpand_fyi(tbl) /* can be defined to log noexpand */ +#endif +#ifndef uthash_expand_fyi +#define uthash_expand_fyi(tbl) /* can be defined to log expands */ +#endif + +#ifndef HASH_NONFATAL_OOM +#define HASH_NONFATAL_OOM 0 +#endif + +#if HASH_NONFATAL_OOM +/* malloc failures can be recovered from */ + +#ifndef uthash_nonfatal_oom +#define uthash_nonfatal_oom(obj) do {} while (0) /* non-fatal OOM error */ +#endif + +#define HASH_RECORD_OOM(oomed) do { (oomed) = 1; } while (0) +#define IF_HASH_NONFATAL_OOM(x) x + +#else +/* malloc failures result in lost memory, hash tables are unusable */ + +#ifndef uthash_fatal +#define uthash_fatal(msg) exit(-1) /* fatal OOM error */ +#endif + +#define HASH_RECORD_OOM(oomed) uthash_fatal("out of memory") +#define IF_HASH_NONFATAL_OOM(x) + +#endif + +/* initial number of buckets */ +#define HASH_INITIAL_NUM_BUCKETS 32U /* initial number of buckets */ +#define HASH_INITIAL_NUM_BUCKETS_LOG2 5U /* lg2 of initial number of buckets */ +#define HASH_BKT_CAPACITY_THRESH 10U /* expand when bucket count reaches */ + +/* calculate the element whose hash handle address is hhp */ +#define ELMT_FROM_HH(tbl,hhp) ((void*)(((char*)(hhp)) - ((tbl)->hho))) +/* calculate the hash handle from element address elp */ +#define HH_FROM_ELMT(tbl,elp) ((UT_hash_handle *)(((char*)(elp)) + ((tbl)->hho))) + +#define HASH_ROLLBACK_BKT(hh, head, itemptrhh) \ +do { \ + struct UT_hash_handle *_hd_hh_item = (itemptrhh); \ + unsigned _hd_bkt; \ + HASH_TO_BKT(_hd_hh_item->hashv, (head)->hh.tbl->num_buckets, _hd_bkt); \ + (head)->hh.tbl->buckets[_hd_bkt].count++; \ + _hd_hh_item->hh_next = NULL; \ + _hd_hh_item->hh_prev = NULL; \ +} while (0) + +#define HASH_VALUE(keyptr,keylen,hashv) \ +do { \ + HASH_FCN(keyptr, keylen, hashv); \ +} while (0) + +#define HASH_FIND_BYHASHVALUE(hh,head,keyptr,keylen,hashval,out) \ +do { \ + (out) = NULL; \ + if (head) { \ + unsigned _hf_bkt; \ + HASH_TO_BKT(hashval, (head)->hh.tbl->num_buckets, _hf_bkt); \ + if (HASH_BLOOM_TEST((head)->hh.tbl, hashval) != 0) { \ + HASH_FIND_IN_BKT((head)->hh.tbl, hh, (head)->hh.tbl->buckets[ _hf_bkt ], keyptr, keylen, hashval, out); \ + } \ + } \ +} while (0) + +#define HASH_FIND(hh,head,keyptr,keylen,out) \ +do { \ + (out) = NULL; \ + if (head) { \ + unsigned _hf_hashv; \ + HASH_VALUE(keyptr, keylen, _hf_hashv); \ + HASH_FIND_BYHASHVALUE(hh, head, keyptr, keylen, _hf_hashv, out); \ + } \ +} while (0) + +#ifdef HASH_BLOOM +#define HASH_BLOOM_BITLEN (1UL << HASH_BLOOM) +#define HASH_BLOOM_BYTELEN (HASH_BLOOM_BITLEN/8UL) + (((HASH_BLOOM_BITLEN%8UL)!=0UL) ? 1UL : 0UL) +#define HASH_BLOOM_MAKE(tbl,oomed) \ +do { \ + (tbl)->bloom_nbits = HASH_BLOOM; \ + (tbl)->bloom_bv = (uint8_t*)uthash_malloc(HASH_BLOOM_BYTELEN); \ + if (!(tbl)->bloom_bv) { \ + HASH_RECORD_OOM(oomed); \ + } else { \ + uthash_bzero((tbl)->bloom_bv, HASH_BLOOM_BYTELEN); \ + (tbl)->bloom_sig = HASH_BLOOM_SIGNATURE; \ + } \ +} while (0) + +#define HASH_BLOOM_FREE(tbl) \ +do { \ + uthash_free((tbl)->bloom_bv, HASH_BLOOM_BYTELEN); \ +} while (0) + +#define HASH_BLOOM_BITSET(bv,idx) (bv[(idx)/8U] |= (1U << ((idx)%8U))) +#define HASH_BLOOM_BITTEST(bv,idx) (bv[(idx)/8U] & (1U << ((idx)%8U))) + +#define HASH_BLOOM_ADD(tbl,hashv) \ + HASH_BLOOM_BITSET((tbl)->bloom_bv, ((hashv) & (uint32_t)((1UL << (tbl)->bloom_nbits) - 1U))) + +#define HASH_BLOOM_TEST(tbl,hashv) \ + HASH_BLOOM_BITTEST((tbl)->bloom_bv, ((hashv) & (uint32_t)((1UL << (tbl)->bloom_nbits) - 1U))) + +#else +#define HASH_BLOOM_MAKE(tbl,oomed) +#define HASH_BLOOM_FREE(tbl) +#define HASH_BLOOM_ADD(tbl,hashv) +#define HASH_BLOOM_TEST(tbl,hashv) (1) +#define HASH_BLOOM_BYTELEN 0U +#endif + +#define HASH_MAKE_TABLE(hh,head,oomed) \ +do { \ + (head)->hh.tbl = (UT_hash_table*)uthash_malloc(sizeof(UT_hash_table)); \ + if (!(head)->hh.tbl) { \ + HASH_RECORD_OOM(oomed); \ + } else { \ + uthash_bzero((head)->hh.tbl, sizeof(UT_hash_table)); \ + (head)->hh.tbl->tail = &((head)->hh); \ + (head)->hh.tbl->num_buckets = HASH_INITIAL_NUM_BUCKETS; \ + (head)->hh.tbl->log2_num_buckets = HASH_INITIAL_NUM_BUCKETS_LOG2; \ + (head)->hh.tbl->hho = (char*)(&(head)->hh) - (char*)(head); \ + (head)->hh.tbl->buckets = (UT_hash_bucket*)uthash_malloc( \ + HASH_INITIAL_NUM_BUCKETS * sizeof(struct UT_hash_bucket)); \ + (head)->hh.tbl->signature = HASH_SIGNATURE; \ + if (!(head)->hh.tbl->buckets) { \ + HASH_RECORD_OOM(oomed); \ + uthash_free((head)->hh.tbl, sizeof(UT_hash_table)); \ + } else { \ + uthash_bzero((head)->hh.tbl->buckets, \ + HASH_INITIAL_NUM_BUCKETS * sizeof(struct UT_hash_bucket)); \ + HASH_BLOOM_MAKE((head)->hh.tbl, oomed); \ + IF_HASH_NONFATAL_OOM( \ + if (oomed) { \ + uthash_free((head)->hh.tbl->buckets, \ + HASH_INITIAL_NUM_BUCKETS*sizeof(struct UT_hash_bucket)); \ + uthash_free((head)->hh.tbl, sizeof(UT_hash_table)); \ + } \ + ) \ + } \ + } \ +} while (0) + +#define HASH_REPLACE_BYHASHVALUE_INORDER(hh,head,fieldname,keylen_in,hashval,add,replaced,cmpfcn) \ +do { \ + (replaced) = NULL; \ + HASH_FIND_BYHASHVALUE(hh, head, &((add)->fieldname), keylen_in, hashval, replaced); \ + if (replaced) { \ + HASH_DELETE(hh, head, replaced); \ + } \ + HASH_ADD_KEYPTR_BYHASHVALUE_INORDER(hh, head, &((add)->fieldname), keylen_in, hashval, add, cmpfcn); \ +} while (0) + +#define HASH_REPLACE_BYHASHVALUE(hh,head,fieldname,keylen_in,hashval,add,replaced) \ +do { \ + (replaced) = NULL; \ + HASH_FIND_BYHASHVALUE(hh, head, &((add)->fieldname), keylen_in, hashval, replaced); \ + if (replaced) { \ + HASH_DELETE(hh, head, replaced); \ + } \ + HASH_ADD_KEYPTR_BYHASHVALUE(hh, head, &((add)->fieldname), keylen_in, hashval, add); \ +} while (0) + +#define HASH_REPLACE(hh,head,fieldname,keylen_in,add,replaced) \ +do { \ + unsigned _hr_hashv; \ + HASH_VALUE(&((add)->fieldname), keylen_in, _hr_hashv); \ + HASH_REPLACE_BYHASHVALUE(hh, head, fieldname, keylen_in, _hr_hashv, add, replaced); \ +} while (0) + +#define HASH_REPLACE_INORDER(hh,head,fieldname,keylen_in,add,replaced,cmpfcn) \ +do { \ + unsigned _hr_hashv; \ + HASH_VALUE(&((add)->fieldname), keylen_in, _hr_hashv); \ + HASH_REPLACE_BYHASHVALUE_INORDER(hh, head, fieldname, keylen_in, _hr_hashv, add, replaced, cmpfcn); \ +} while (0) + +#define HASH_APPEND_LIST(hh, head, add) \ +do { \ + (add)->hh.next = NULL; \ + (add)->hh.prev = ELMT_FROM_HH((head)->hh.tbl, (head)->hh.tbl->tail); \ + (head)->hh.tbl->tail->next = (add); \ + (head)->hh.tbl->tail = &((add)->hh); \ +} while (0) + +#define HASH_AKBI_INNER_LOOP(hh,head,add,cmpfcn) \ +do { \ + do { \ + if (cmpfcn(DECLTYPE(head)(_hs_iter), add) > 0) { \ + break; \ + } \ + } while ((_hs_iter = HH_FROM_ELMT((head)->hh.tbl, _hs_iter)->next)); \ +} while (0) + +#ifdef NO_DECLTYPE +#undef HASH_AKBI_INNER_LOOP +#define HASH_AKBI_INNER_LOOP(hh,head,add,cmpfcn) \ +do { \ + char *_hs_saved_head = (char*)(head); \ + do { \ + DECLTYPE_ASSIGN(head, _hs_iter); \ + if (cmpfcn(head, add) > 0) { \ + DECLTYPE_ASSIGN(head, _hs_saved_head); \ + break; \ + } \ + DECLTYPE_ASSIGN(head, _hs_saved_head); \ + } while ((_hs_iter = HH_FROM_ELMT((head)->hh.tbl, _hs_iter)->next)); \ +} while (0) +#endif + +#if HASH_NONFATAL_OOM + +#define HASH_ADD_TO_TABLE(hh,head,keyptr,keylen_in,hashval,add,oomed) \ +do { \ + if (!(oomed)) { \ + unsigned _ha_bkt; \ + (head)->hh.tbl->num_items++; \ + HASH_TO_BKT(hashval, (head)->hh.tbl->num_buckets, _ha_bkt); \ + HASH_ADD_TO_BKT((head)->hh.tbl->buckets[_ha_bkt], hh, &(add)->hh, oomed); \ + if (oomed) { \ + HASH_ROLLBACK_BKT(hh, head, &(add)->hh); \ + HASH_DELETE_HH(hh, head, &(add)->hh); \ + (add)->hh.tbl = NULL; \ + uthash_nonfatal_oom(add); \ + } else { \ + HASH_BLOOM_ADD((head)->hh.tbl, hashval); \ + HASH_EMIT_KEY(hh, head, keyptr, keylen_in); \ + } \ + } else { \ + (add)->hh.tbl = NULL; \ + uthash_nonfatal_oom(add); \ + } \ +} while (0) + +#else + +#define HASH_ADD_TO_TABLE(hh,head,keyptr,keylen_in,hashval,add,oomed) \ +do { \ + unsigned _ha_bkt; \ + (head)->hh.tbl->num_items++; \ + HASH_TO_BKT(hashval, (head)->hh.tbl->num_buckets, _ha_bkt); \ + HASH_ADD_TO_BKT((head)->hh.tbl->buckets[_ha_bkt], hh, &(add)->hh, oomed); \ + HASH_BLOOM_ADD((head)->hh.tbl, hashval); \ + HASH_EMIT_KEY(hh, head, keyptr, keylen_in); \ +} while (0) + +#endif + + +#define HASH_ADD_KEYPTR_BYHASHVALUE_INORDER(hh,head,keyptr,keylen_in,hashval,add,cmpfcn) \ +do { \ + IF_HASH_NONFATAL_OOM( int _ha_oomed = 0; ) \ + (add)->hh.hashv = (hashval); \ + (add)->hh.key = (char*) (keyptr); \ + (add)->hh.keylen = (unsigned) (keylen_in); \ + if (!(head)) { \ + (add)->hh.next = NULL; \ + (add)->hh.prev = NULL; \ + HASH_MAKE_TABLE(hh, add, _ha_oomed); \ + IF_HASH_NONFATAL_OOM( if (!_ha_oomed) { ) \ + (head) = (add); \ + IF_HASH_NONFATAL_OOM( } ) \ + } else { \ + void *_hs_iter = (head); \ + (add)->hh.tbl = (head)->hh.tbl; \ + HASH_AKBI_INNER_LOOP(hh, head, add, cmpfcn); \ + if (_hs_iter) { \ + (add)->hh.next = _hs_iter; \ + if (((add)->hh.prev = HH_FROM_ELMT((head)->hh.tbl, _hs_iter)->prev)) { \ + HH_FROM_ELMT((head)->hh.tbl, (add)->hh.prev)->next = (add); \ + } else { \ + (head) = (add); \ + } \ + HH_FROM_ELMT((head)->hh.tbl, _hs_iter)->prev = (add); \ + } else { \ + HASH_APPEND_LIST(hh, head, add); \ + } \ + } \ + HASH_ADD_TO_TABLE(hh, head, keyptr, keylen_in, hashval, add, _ha_oomed); \ + HASH_FSCK(hh, head, "HASH_ADD_KEYPTR_BYHASHVALUE_INORDER"); \ +} while (0) + +#define HASH_ADD_KEYPTR_INORDER(hh,head,keyptr,keylen_in,add,cmpfcn) \ +do { \ + unsigned _hs_hashv; \ + HASH_VALUE(keyptr, keylen_in, _hs_hashv); \ + HASH_ADD_KEYPTR_BYHASHVALUE_INORDER(hh, head, keyptr, keylen_in, _hs_hashv, add, cmpfcn); \ +} while (0) + +#define HASH_ADD_BYHASHVALUE_INORDER(hh,head,fieldname,keylen_in,hashval,add,cmpfcn) \ + HASH_ADD_KEYPTR_BYHASHVALUE_INORDER(hh, head, &((add)->fieldname), keylen_in, hashval, add, cmpfcn) + +#define HASH_ADD_INORDER(hh,head,fieldname,keylen_in,add,cmpfcn) \ + HASH_ADD_KEYPTR_INORDER(hh, head, &((add)->fieldname), keylen_in, add, cmpfcn) + +#define HASH_ADD_KEYPTR_BYHASHVALUE(hh,head,keyptr,keylen_in,hashval,add) \ +do { \ + IF_HASH_NONFATAL_OOM( int _ha_oomed = 0; ) \ + (add)->hh.hashv = (hashval); \ + (add)->hh.key = (char*) (keyptr); \ + (add)->hh.keylen = (unsigned) (keylen_in); \ + if (!(head)) { \ + (add)->hh.next = NULL; \ + (add)->hh.prev = NULL; \ + HASH_MAKE_TABLE(hh, add, _ha_oomed); \ + IF_HASH_NONFATAL_OOM( if (!_ha_oomed) { ) \ + (head) = (add); \ + IF_HASH_NONFATAL_OOM( } ) \ + } else { \ + (add)->hh.tbl = (head)->hh.tbl; \ + HASH_APPEND_LIST(hh, head, add); \ + } \ + HASH_ADD_TO_TABLE(hh, head, keyptr, keylen_in, hashval, add, _ha_oomed); \ + HASH_FSCK(hh, head, "HASH_ADD_KEYPTR_BYHASHVALUE"); \ +} while (0) + +#define HASH_ADD_KEYPTR(hh,head,keyptr,keylen_in,add) \ +do { \ + unsigned _ha_hashv; \ + HASH_VALUE(keyptr, keylen_in, _ha_hashv); \ + HASH_ADD_KEYPTR_BYHASHVALUE(hh, head, keyptr, keylen_in, _ha_hashv, add); \ +} while (0) + +#define HASH_ADD_BYHASHVALUE(hh,head,fieldname,keylen_in,hashval,add) \ + HASH_ADD_KEYPTR_BYHASHVALUE(hh, head, &((add)->fieldname), keylen_in, hashval, add) + +#define HASH_ADD(hh,head,fieldname,keylen_in,add) \ + HASH_ADD_KEYPTR(hh, head, &((add)->fieldname), keylen_in, add) + +#define HASH_TO_BKT(hashv,num_bkts,bkt) \ +do { \ + bkt = ((hashv) & ((num_bkts) - 1U)); \ +} while (0) + +/* delete "delptr" from the hash table. + * "the usual" patch-up process for the app-order doubly-linked-list. + * The use of _hd_hh_del below deserves special explanation. + * These used to be expressed using (delptr) but that led to a bug + * if someone used the same symbol for the head and deletee, like + * HASH_DELETE(hh,users,users); + * We want that to work, but by changing the head (users) below + * we were forfeiting our ability to further refer to the deletee (users) + * in the patch-up process. Solution: use scratch space to + * copy the deletee pointer, then the latter references are via that + * scratch pointer rather than through the repointed (users) symbol. + */ +#define HASH_DELETE(hh,head,delptr) \ + HASH_DELETE_HH(hh, head, &(delptr)->hh) + +#define HASH_DELETE_HH(hh,head,delptrhh) \ +do { \ + struct UT_hash_handle *_hd_hh_del = (delptrhh); \ + if ((_hd_hh_del->prev == NULL) && (_hd_hh_del->next == NULL)) { \ + HASH_BLOOM_FREE((head)->hh.tbl); \ + uthash_free((head)->hh.tbl->buckets, \ + (head)->hh.tbl->num_buckets * sizeof(struct UT_hash_bucket)); \ + uthash_free((head)->hh.tbl, sizeof(UT_hash_table)); \ + (head) = NULL; \ + } else { \ + unsigned _hd_bkt; \ + if (_hd_hh_del == (head)->hh.tbl->tail) { \ + (head)->hh.tbl->tail = HH_FROM_ELMT((head)->hh.tbl, _hd_hh_del->prev); \ + } \ + if (_hd_hh_del->prev != NULL) { \ + HH_FROM_ELMT((head)->hh.tbl, _hd_hh_del->prev)->next = _hd_hh_del->next; \ + } else { \ + DECLTYPE_ASSIGN(head, _hd_hh_del->next); \ + } \ + if (_hd_hh_del->next != NULL) { \ + HH_FROM_ELMT((head)->hh.tbl, _hd_hh_del->next)->prev = _hd_hh_del->prev; \ + } \ + HASH_TO_BKT(_hd_hh_del->hashv, (head)->hh.tbl->num_buckets, _hd_bkt); \ + HASH_DEL_IN_BKT((head)->hh.tbl->buckets[_hd_bkt], _hd_hh_del); \ + (head)->hh.tbl->num_items--; \ + } \ + HASH_FSCK(hh, head, "HASH_DELETE_HH"); \ +} while (0) + +/* convenience forms of HASH_FIND/HASH_ADD/HASH_DEL */ +#define HASH_FIND_STR(head,findstr,out) \ +do { \ + unsigned _uthash_hfstr_keylen = (unsigned)uthash_strlen(findstr); \ + HASH_FIND(hh, head, findstr, _uthash_hfstr_keylen, out); \ +} while (0) +#define HASH_ADD_STR(head,strfield,add) \ +do { \ + unsigned _uthash_hastr_keylen = (unsigned)uthash_strlen((add)->strfield); \ + HASH_ADD(hh, head, strfield[0], _uthash_hastr_keylen, add); \ +} while (0) +#define HASH_REPLACE_STR(head,strfield,add,replaced) \ +do { \ + unsigned _uthash_hrstr_keylen = (unsigned)uthash_strlen((add)->strfield); \ + HASH_REPLACE(hh, head, strfield[0], _uthash_hrstr_keylen, add, replaced); \ +} while (0) +#define HASH_FIND_INT(head,findint,out) \ + HASH_FIND(hh,head,findint,sizeof(int),out) +#define HASH_ADD_INT(head,intfield,add) \ + HASH_ADD(hh,head,intfield,sizeof(int),add) +#define HASH_REPLACE_INT(head,intfield,add,replaced) \ + HASH_REPLACE(hh,head,intfield,sizeof(int),add,replaced) +#define HASH_FIND_PTR(head,findptr,out) \ + HASH_FIND(hh,head,findptr,sizeof(void *),out) +#define HASH_ADD_PTR(head,ptrfield,add) \ + HASH_ADD(hh,head,ptrfield,sizeof(void *),add) +#define HASH_REPLACE_PTR(head,ptrfield,add,replaced) \ + HASH_REPLACE(hh,head,ptrfield,sizeof(void *),add,replaced) +#define HASH_DEL(head,delptr) \ + HASH_DELETE(hh,head,delptr) + +/* HASH_FSCK checks hash integrity on every add/delete when HASH_DEBUG is defined. + * This is for uthash developer only; it compiles away if HASH_DEBUG isn't defined. + */ +#ifdef HASH_DEBUG +#define HASH_OOPS(...) do { fprintf(stderr,__VA_ARGS__); exit(-1); } while (0) +#define HASH_FSCK(hh,head,where) \ +do { \ + struct UT_hash_handle *_thh; \ + if (head) { \ + unsigned _bkt_i; \ + unsigned _count = 0; \ + char *_prev; \ + for (_bkt_i = 0; _bkt_i < (head)->hh.tbl->num_buckets; ++_bkt_i) { \ + unsigned _bkt_count = 0; \ + _thh = (head)->hh.tbl->buckets[_bkt_i].hh_head; \ + _prev = NULL; \ + while (_thh) { \ + if (_prev != (char*)(_thh->hh_prev)) { \ + HASH_OOPS("%s: invalid hh_prev %p, actual %p\n", \ + (where), (void*)_thh->hh_prev, (void*)_prev); \ + } \ + _bkt_count++; \ + _prev = (char*)(_thh); \ + _thh = _thh->hh_next; \ + } \ + _count += _bkt_count; \ + if ((head)->hh.tbl->buckets[_bkt_i].count != _bkt_count) { \ + HASH_OOPS("%s: invalid bucket count %u, actual %u\n", \ + (where), (head)->hh.tbl->buckets[_bkt_i].count, _bkt_count); \ + } \ + } \ + if (_count != (head)->hh.tbl->num_items) { \ + HASH_OOPS("%s: invalid hh item count %u, actual %u\n", \ + (where), (head)->hh.tbl->num_items, _count); \ + } \ + _count = 0; \ + _prev = NULL; \ + _thh = &(head)->hh; \ + while (_thh) { \ + _count++; \ + if (_prev != (char*)_thh->prev) { \ + HASH_OOPS("%s: invalid prev %p, actual %p\n", \ + (where), (void*)_thh->prev, (void*)_prev); \ + } \ + _prev = (char*)ELMT_FROM_HH((head)->hh.tbl, _thh); \ + _thh = (_thh->next ? HH_FROM_ELMT((head)->hh.tbl, _thh->next) : NULL); \ + } \ + if (_count != (head)->hh.tbl->num_items) { \ + HASH_OOPS("%s: invalid app item count %u, actual %u\n", \ + (where), (head)->hh.tbl->num_items, _count); \ + } \ + } \ +} while (0) +#else +#define HASH_FSCK(hh,head,where) +#endif + +/* When compiled with -DHASH_EMIT_KEYS, length-prefixed keys are emitted to + * the descriptor to which this macro is defined for tuning the hash function. + * The app can #include to get the prototype for write(2). */ +#ifdef HASH_EMIT_KEYS +#define HASH_EMIT_KEY(hh,head,keyptr,fieldlen) \ +do { \ + unsigned _klen = fieldlen; \ + write(HASH_EMIT_KEYS, &_klen, sizeof(_klen)); \ + write(HASH_EMIT_KEYS, keyptr, (unsigned long)fieldlen); \ +} while (0) +#else +#define HASH_EMIT_KEY(hh,head,keyptr,fieldlen) +#endif + +/* default to Jenkin's hash unless overridden e.g. DHASH_FUNCTION=HASH_SAX */ +#ifdef HASH_FUNCTION +#define HASH_FCN HASH_FUNCTION +#else +#define HASH_FCN HASH_JEN +#endif + +/* The Bernstein hash function, used in Perl prior to v5.6. Note (x<<5+x)=x*33. */ +#define HASH_BER(key,keylen,hashv) \ +do { \ + unsigned _hb_keylen = (unsigned)keylen; \ + const unsigned char *_hb_key = (const unsigned char*)(key); \ + (hashv) = 0; \ + while (_hb_keylen-- != 0U) { \ + (hashv) = (((hashv) << 5) + (hashv)) + *_hb_key++; \ + } \ +} while (0) + + +/* SAX/FNV/OAT/JEN hash functions are macro variants of those listed at + * http://eternallyconfuzzled.com/tuts/algorithms/jsw_tut_hashing.aspx */ +#define HASH_SAX(key,keylen,hashv) \ +do { \ + unsigned _sx_i; \ + const unsigned char *_hs_key = (const unsigned char*)(key); \ + hashv = 0; \ + for (_sx_i=0; _sx_i < keylen; _sx_i++) { \ + hashv ^= (hashv << 5) + (hashv >> 2) + _hs_key[_sx_i]; \ + } \ +} while (0) +/* FNV-1a variation */ +#define HASH_FNV(key,keylen,hashv) \ +do { \ + unsigned _fn_i; \ + const unsigned char *_hf_key = (const unsigned char*)(key); \ + (hashv) = 2166136261U; \ + for (_fn_i=0; _fn_i < keylen; _fn_i++) { \ + hashv = hashv ^ _hf_key[_fn_i]; \ + hashv = hashv * 16777619U; \ + } \ +} while (0) + +#define HASH_OAT(key,keylen,hashv) \ +do { \ + unsigned _ho_i; \ + const unsigned char *_ho_key=(const unsigned char*)(key); \ + hashv = 0; \ + for(_ho_i=0; _ho_i < keylen; _ho_i++) { \ + hashv += _ho_key[_ho_i]; \ + hashv += (hashv << 10); \ + hashv ^= (hashv >> 6); \ + } \ + hashv += (hashv << 3); \ + hashv ^= (hashv >> 11); \ + hashv += (hashv << 15); \ +} while (0) + +#define HASH_JEN_MIX(a,b,c) \ +do { \ + a -= b; a -= c; a ^= ( c >> 13 ); \ + b -= c; b -= a; b ^= ( a << 8 ); \ + c -= a; c -= b; c ^= ( b >> 13 ); \ + a -= b; a -= c; a ^= ( c >> 12 ); \ + b -= c; b -= a; b ^= ( a << 16 ); \ + c -= a; c -= b; c ^= ( b >> 5 ); \ + a -= b; a -= c; a ^= ( c >> 3 ); \ + b -= c; b -= a; b ^= ( a << 10 ); \ + c -= a; c -= b; c ^= ( b >> 15 ); \ +} while (0) + +#define HASH_JEN(key,keylen,hashv) \ +do { \ + unsigned _hj_i,_hj_j,_hj_k; \ + unsigned const char *_hj_key=(unsigned const char*)(key); \ + hashv = 0xfeedbeefu; \ + _hj_i = _hj_j = 0x9e3779b9u; \ + _hj_k = (unsigned)(keylen); \ + while (_hj_k >= 12U) { \ + _hj_i += (_hj_key[0] + ( (unsigned)_hj_key[1] << 8 ) \ + + ( (unsigned)_hj_key[2] << 16 ) \ + + ( (unsigned)_hj_key[3] << 24 ) ); \ + _hj_j += (_hj_key[4] + ( (unsigned)_hj_key[5] << 8 ) \ + + ( (unsigned)_hj_key[6] << 16 ) \ + + ( (unsigned)_hj_key[7] << 24 ) ); \ + hashv += (_hj_key[8] + ( (unsigned)_hj_key[9] << 8 ) \ + + ( (unsigned)_hj_key[10] << 16 ) \ + + ( (unsigned)_hj_key[11] << 24 ) ); \ + \ + HASH_JEN_MIX(_hj_i, _hj_j, hashv); \ + \ + _hj_key += 12; \ + _hj_k -= 12U; \ + } \ + hashv += (unsigned)(keylen); \ + switch ( _hj_k ) { \ + case 11: hashv += ( (unsigned)_hj_key[10] << 24 ); /* FALLTHROUGH */ \ + case 10: hashv += ( (unsigned)_hj_key[9] << 16 ); /* FALLTHROUGH */ \ + case 9: hashv += ( (unsigned)_hj_key[8] << 8 ); /* FALLTHROUGH */ \ + case 8: _hj_j += ( (unsigned)_hj_key[7] << 24 ); /* FALLTHROUGH */ \ + case 7: _hj_j += ( (unsigned)_hj_key[6] << 16 ); /* FALLTHROUGH */ \ + case 6: _hj_j += ( (unsigned)_hj_key[5] << 8 ); /* FALLTHROUGH */ \ + case 5: _hj_j += _hj_key[4]; /* FALLTHROUGH */ \ + case 4: _hj_i += ( (unsigned)_hj_key[3] << 24 ); /* FALLTHROUGH */ \ + case 3: _hj_i += ( (unsigned)_hj_key[2] << 16 ); /* FALLTHROUGH */ \ + case 2: _hj_i += ( (unsigned)_hj_key[1] << 8 ); /* FALLTHROUGH */ \ + case 1: _hj_i += _hj_key[0]; \ + } \ + HASH_JEN_MIX(_hj_i, _hj_j, hashv); \ +} while (0) + +/* The Paul Hsieh hash function */ +#undef get16bits +#if (defined(__GNUC__) && defined(__i386__)) || defined(__WATCOMC__) \ + || defined(_MSC_VER) || defined (__BORLANDC__) || defined (__TURBOC__) +#define get16bits(d) (*((const uint16_t *) (d))) +#endif + +#if !defined (get16bits) +#define get16bits(d) ((((uint32_t)(((const uint8_t *)(d))[1])) << 8) \ + +(uint32_t)(((const uint8_t *)(d))[0]) ) +#endif +#define HASH_SFH(key,keylen,hashv) \ +do { \ + unsigned const char *_sfh_key=(unsigned const char*)(key); \ + uint32_t _sfh_tmp, _sfh_len = (uint32_t)keylen; \ + \ + unsigned _sfh_rem = _sfh_len & 3U; \ + _sfh_len >>= 2; \ + hashv = 0xcafebabeu; \ + \ + /* Main loop */ \ + for (;_sfh_len > 0U; _sfh_len--) { \ + hashv += get16bits (_sfh_key); \ + _sfh_tmp = ((uint32_t)(get16bits (_sfh_key+2)) << 11) ^ hashv; \ + hashv = (hashv << 16) ^ _sfh_tmp; \ + _sfh_key += 2U*sizeof (uint16_t); \ + hashv += hashv >> 11; \ + } \ + \ + /* Handle end cases */ \ + switch (_sfh_rem) { \ + case 3: hashv += get16bits (_sfh_key); \ + hashv ^= hashv << 16; \ + hashv ^= (uint32_t)(_sfh_key[sizeof (uint16_t)]) << 18; \ + hashv += hashv >> 11; \ + break; \ + case 2: hashv += get16bits (_sfh_key); \ + hashv ^= hashv << 11; \ + hashv += hashv >> 17; \ + break; \ + case 1: hashv += *_sfh_key; \ + hashv ^= hashv << 10; \ + hashv += hashv >> 1; \ + } \ + \ + /* Force "avalanching" of final 127 bits */ \ + hashv ^= hashv << 3; \ + hashv += hashv >> 5; \ + hashv ^= hashv << 4; \ + hashv += hashv >> 17; \ + hashv ^= hashv << 25; \ + hashv += hashv >> 6; \ +} while (0) + +#ifdef HASH_USING_NO_STRICT_ALIASING +/* The MurmurHash exploits some CPU's (x86,x86_64) tolerance for unaligned reads. + * For other types of CPU's (e.g. Sparc) an unaligned read causes a bus error. + * MurmurHash uses the faster approach only on CPU's where we know it's safe. + * + * Note the preprocessor built-in defines can be emitted using: + * + * gcc -m64 -dM -E - < /dev/null (on gcc) + * cc -## a.c (where a.c is a simple test file) (Sun Studio) + */ +#if (defined(__i386__) || defined(__x86_64__) || defined(_M_IX86)) +#define MUR_GETBLOCK(p,i) p[i] +#else /* non intel */ +#define MUR_PLUS0_ALIGNED(p) (((unsigned long)p & 3UL) == 0UL) +#define MUR_PLUS1_ALIGNED(p) (((unsigned long)p & 3UL) == 1UL) +#define MUR_PLUS2_ALIGNED(p) (((unsigned long)p & 3UL) == 2UL) +#define MUR_PLUS3_ALIGNED(p) (((unsigned long)p & 3UL) == 3UL) +#define WP(p) ((uint32_t*)((unsigned long)(p) & ~3UL)) +#if (defined(__BIG_ENDIAN__) || defined(SPARC) || defined(__ppc__) || defined(__ppc64__)) +#define MUR_THREE_ONE(p) ((((*WP(p))&0x00ffffff) << 8) | (((*(WP(p)+1))&0xff000000) >> 24)) +#define MUR_TWO_TWO(p) ((((*WP(p))&0x0000ffff) <<16) | (((*(WP(p)+1))&0xffff0000) >> 16)) +#define MUR_ONE_THREE(p) ((((*WP(p))&0x000000ff) <<24) | (((*(WP(p)+1))&0xffffff00) >> 8)) +#else /* assume little endian non-intel */ +#define MUR_THREE_ONE(p) ((((*WP(p))&0xffffff00) >> 8) | (((*(WP(p)+1))&0x000000ff) << 24)) +#define MUR_TWO_TWO(p) ((((*WP(p))&0xffff0000) >>16) | (((*(WP(p)+1))&0x0000ffff) << 16)) +#define MUR_ONE_THREE(p) ((((*WP(p))&0xff000000) >>24) | (((*(WP(p)+1))&0x00ffffff) << 8)) +#endif +#define MUR_GETBLOCK(p,i) (MUR_PLUS0_ALIGNED(p) ? ((p)[i]) : \ + (MUR_PLUS1_ALIGNED(p) ? MUR_THREE_ONE(p) : \ + (MUR_PLUS2_ALIGNED(p) ? MUR_TWO_TWO(p) : \ + MUR_ONE_THREE(p)))) +#endif +#define MUR_ROTL32(x,r) (((x) << (r)) | ((x) >> (32 - (r)))) +#define MUR_FMIX(_h) \ +do { \ + _h ^= _h >> 16; \ + _h *= 0x85ebca6bu; \ + _h ^= _h >> 13; \ + _h *= 0xc2b2ae35u; \ + _h ^= _h >> 16; \ +} while (0) + +#define HASH_MUR(key,keylen,hashv) \ +do { \ + const uint8_t *_mur_data = (const uint8_t*)(key); \ + const int _mur_nblocks = (int)(keylen) / 4; \ + uint32_t _mur_h1 = 0xf88D5353u; \ + uint32_t _mur_c1 = 0xcc9e2d51u; \ + uint32_t _mur_c2 = 0x1b873593u; \ + uint32_t _mur_k1 = 0; \ + const uint8_t *_mur_tail; \ + const uint32_t *_mur_blocks = (const uint32_t*)(_mur_data+(_mur_nblocks*4)); \ + int _mur_i; \ + for (_mur_i = -_mur_nblocks; _mur_i != 0; _mur_i++) { \ + _mur_k1 = MUR_GETBLOCK(_mur_blocks,_mur_i); \ + _mur_k1 *= _mur_c1; \ + _mur_k1 = MUR_ROTL32(_mur_k1,15); \ + _mur_k1 *= _mur_c2; \ + \ + _mur_h1 ^= _mur_k1; \ + _mur_h1 = MUR_ROTL32(_mur_h1,13); \ + _mur_h1 = (_mur_h1*5U) + 0xe6546b64u; \ + } \ + _mur_tail = (const uint8_t*)(_mur_data + (_mur_nblocks*4)); \ + _mur_k1=0; \ + switch ((keylen) & 3U) { \ + case 0: break; \ + case 3: _mur_k1 ^= (uint32_t)_mur_tail[2] << 16; /* FALLTHROUGH */ \ + case 2: _mur_k1 ^= (uint32_t)_mur_tail[1] << 8; /* FALLTHROUGH */ \ + case 1: _mur_k1 ^= (uint32_t)_mur_tail[0]; \ + _mur_k1 *= _mur_c1; \ + _mur_k1 = MUR_ROTL32(_mur_k1,15); \ + _mur_k1 *= _mur_c2; \ + _mur_h1 ^= _mur_k1; \ + } \ + _mur_h1 ^= (uint32_t)(keylen); \ + MUR_FMIX(_mur_h1); \ + hashv = _mur_h1; \ +} while (0) +#endif /* HASH_USING_NO_STRICT_ALIASING */ + +/* iterate over items in a known bucket to find desired item */ +#define HASH_FIND_IN_BKT(tbl,hh,head,keyptr,keylen_in,hashval,out) \ +do { \ + if ((head).hh_head != NULL) { \ + DECLTYPE_ASSIGN(out, ELMT_FROM_HH(tbl, (head).hh_head)); \ + } else { \ + (out) = NULL; \ + } \ + while ((out) != NULL) { \ + if ((out)->hh.hashv == (hashval) && (out)->hh.keylen == (keylen_in)) { \ + if (HASH_KEYCMP((out)->hh.key, keyptr, keylen_in) == 0) { \ + break; \ + } \ + } \ + if ((out)->hh.hh_next != NULL) { \ + DECLTYPE_ASSIGN(out, ELMT_FROM_HH(tbl, (out)->hh.hh_next)); \ + } else { \ + (out) = NULL; \ + } \ + } \ +} while (0) + +/* add an item to a bucket */ +#define HASH_ADD_TO_BKT(head,hh,addhh,oomed) \ +do { \ + UT_hash_bucket *_ha_head = &(head); \ + _ha_head->count++; \ + (addhh)->hh_next = _ha_head->hh_head; \ + (addhh)->hh_prev = NULL; \ + if (_ha_head->hh_head != NULL) { \ + _ha_head->hh_head->hh_prev = (addhh); \ + } \ + _ha_head->hh_head = (addhh); \ + if ((_ha_head->count >= ((_ha_head->expand_mult + 1U) * HASH_BKT_CAPACITY_THRESH)) \ + && !(addhh)->tbl->noexpand) { \ + HASH_EXPAND_BUCKETS(addhh,(addhh)->tbl, oomed); \ + IF_HASH_NONFATAL_OOM( \ + if (oomed) { \ + HASH_DEL_IN_BKT(head,addhh); \ + } \ + ) \ + } \ +} while (0) + +/* remove an item from a given bucket */ +#define HASH_DEL_IN_BKT(head,delhh) \ +do { \ + UT_hash_bucket *_hd_head = &(head); \ + _hd_head->count--; \ + if (_hd_head->hh_head == (delhh)) { \ + _hd_head->hh_head = (delhh)->hh_next; \ + } \ + if ((delhh)->hh_prev) { \ + (delhh)->hh_prev->hh_next = (delhh)->hh_next; \ + } \ + if ((delhh)->hh_next) { \ + (delhh)->hh_next->hh_prev = (delhh)->hh_prev; \ + } \ +} while (0) + +/* Bucket expansion has the effect of doubling the number of buckets + * and redistributing the items into the new buckets. Ideally the + * items will distribute more or less evenly into the new buckets + * (the extent to which this is true is a measure of the quality of + * the hash function as it applies to the key domain). + * + * With the items distributed into more buckets, the chain length + * (item count) in each bucket is reduced. Thus by expanding buckets + * the hash keeps a bound on the chain length. This bounded chain + * length is the essence of how a hash provides constant time lookup. + * + * The calculation of tbl->ideal_chain_maxlen below deserves some + * explanation. First, keep in mind that we're calculating the ideal + * maximum chain length based on the *new* (doubled) bucket count. + * In fractions this is just n/b (n=number of items,b=new num buckets). + * Since the ideal chain length is an integer, we want to calculate + * ceil(n/b). We don't depend on floating point arithmetic in this + * hash, so to calculate ceil(n/b) with integers we could write + * + * ceil(n/b) = (n/b) + ((n%b)?1:0) + * + * and in fact a previous version of this hash did just that. + * But now we have improved things a bit by recognizing that b is + * always a power of two. We keep its base 2 log handy (call it lb), + * so now we can write this with a bit shift and logical AND: + * + * ceil(n/b) = (n>>lb) + ( (n & (b-1)) ? 1:0) + * + */ +#define HASH_EXPAND_BUCKETS(hh,tbl,oomed) \ +do { \ + unsigned _he_bkt; \ + unsigned _he_bkt_i; \ + struct UT_hash_handle *_he_thh, *_he_hh_nxt; \ + UT_hash_bucket *_he_new_buckets, *_he_newbkt; \ + _he_new_buckets = (UT_hash_bucket*)uthash_malloc( \ + 2UL * (tbl)->num_buckets * sizeof(struct UT_hash_bucket)); \ + if (!_he_new_buckets) { \ + HASH_RECORD_OOM(oomed); \ + } else { \ + uthash_bzero(_he_new_buckets, \ + 2UL * (tbl)->num_buckets * sizeof(struct UT_hash_bucket)); \ + (tbl)->ideal_chain_maxlen = \ + ((tbl)->num_items >> ((tbl)->log2_num_buckets+1U)) + \ + ((((tbl)->num_items & (((tbl)->num_buckets*2U)-1U)) != 0U) ? 1U : 0U); \ + (tbl)->nonideal_items = 0; \ + for (_he_bkt_i = 0; _he_bkt_i < (tbl)->num_buckets; _he_bkt_i++) { \ + _he_thh = (tbl)->buckets[ _he_bkt_i ].hh_head; \ + while (_he_thh != NULL) { \ + _he_hh_nxt = _he_thh->hh_next; \ + HASH_TO_BKT(_he_thh->hashv, (tbl)->num_buckets * 2U, _he_bkt); \ + _he_newbkt = &(_he_new_buckets[_he_bkt]); \ + if (++(_he_newbkt->count) > (tbl)->ideal_chain_maxlen) { \ + (tbl)->nonideal_items++; \ + if (_he_newbkt->count > _he_newbkt->expand_mult * (tbl)->ideal_chain_maxlen) { \ + _he_newbkt->expand_mult++; \ + } \ + } \ + _he_thh->hh_prev = NULL; \ + _he_thh->hh_next = _he_newbkt->hh_head; \ + if (_he_newbkt->hh_head != NULL) { \ + _he_newbkt->hh_head->hh_prev = _he_thh; \ + } \ + _he_newbkt->hh_head = _he_thh; \ + _he_thh = _he_hh_nxt; \ + } \ + } \ + uthash_free((tbl)->buckets, (tbl)->num_buckets * sizeof(struct UT_hash_bucket)); \ + (tbl)->num_buckets *= 2U; \ + (tbl)->log2_num_buckets++; \ + (tbl)->buckets = _he_new_buckets; \ + (tbl)->ineff_expands = ((tbl)->nonideal_items > ((tbl)->num_items >> 1)) ? \ + ((tbl)->ineff_expands+1U) : 0U; \ + if ((tbl)->ineff_expands > 1U) { \ + (tbl)->noexpand = 1; \ + uthash_noexpand_fyi(tbl); \ + } \ + uthash_expand_fyi(tbl); \ + } \ +} while (0) + + +/* This is an adaptation of Simon Tatham's O(n log(n)) mergesort */ +/* Note that HASH_SORT assumes the hash handle name to be hh. + * HASH_SRT was added to allow the hash handle name to be passed in. */ +#define HASH_SORT(head,cmpfcn) HASH_SRT(hh,head,cmpfcn) +#define HASH_SRT(hh,head,cmpfcn) \ +do { \ + unsigned _hs_i; \ + unsigned _hs_looping,_hs_nmerges,_hs_insize,_hs_psize,_hs_qsize; \ + struct UT_hash_handle *_hs_p, *_hs_q, *_hs_e, *_hs_list, *_hs_tail; \ + if (head != NULL) { \ + _hs_insize = 1; \ + _hs_looping = 1; \ + _hs_list = &((head)->hh); \ + while (_hs_looping != 0U) { \ + _hs_p = _hs_list; \ + _hs_list = NULL; \ + _hs_tail = NULL; \ + _hs_nmerges = 0; \ + while (_hs_p != NULL) { \ + _hs_nmerges++; \ + _hs_q = _hs_p; \ + _hs_psize = 0; \ + for (_hs_i = 0; _hs_i < _hs_insize; ++_hs_i) { \ + _hs_psize++; \ + _hs_q = ((_hs_q->next != NULL) ? \ + HH_FROM_ELMT((head)->hh.tbl, _hs_q->next) : NULL); \ + if (_hs_q == NULL) { \ + break; \ + } \ + } \ + _hs_qsize = _hs_insize; \ + while ((_hs_psize != 0U) || ((_hs_qsize != 0U) && (_hs_q != NULL))) { \ + if (_hs_psize == 0U) { \ + _hs_e = _hs_q; \ + _hs_q = ((_hs_q->next != NULL) ? \ + HH_FROM_ELMT((head)->hh.tbl, _hs_q->next) : NULL); \ + _hs_qsize--; \ + } else if ((_hs_qsize == 0U) || (_hs_q == NULL)) { \ + _hs_e = _hs_p; \ + if (_hs_p != NULL) { \ + _hs_p = ((_hs_p->next != NULL) ? \ + HH_FROM_ELMT((head)->hh.tbl, _hs_p->next) : NULL); \ + } \ + _hs_psize--; \ + } else if ((cmpfcn( \ + DECLTYPE(head)(ELMT_FROM_HH((head)->hh.tbl, _hs_p)), \ + DECLTYPE(head)(ELMT_FROM_HH((head)->hh.tbl, _hs_q)) \ + )) <= 0) { \ + _hs_e = _hs_p; \ + if (_hs_p != NULL) { \ + _hs_p = ((_hs_p->next != NULL) ? \ + HH_FROM_ELMT((head)->hh.tbl, _hs_p->next) : NULL); \ + } \ + _hs_psize--; \ + } else { \ + _hs_e = _hs_q; \ + _hs_q = ((_hs_q->next != NULL) ? \ + HH_FROM_ELMT((head)->hh.tbl, _hs_q->next) : NULL); \ + _hs_qsize--; \ + } \ + if ( _hs_tail != NULL ) { \ + _hs_tail->next = ((_hs_e != NULL) ? \ + ELMT_FROM_HH((head)->hh.tbl, _hs_e) : NULL); \ + } else { \ + _hs_list = _hs_e; \ + } \ + if (_hs_e != NULL) { \ + _hs_e->prev = ((_hs_tail != NULL) ? \ + ELMT_FROM_HH((head)->hh.tbl, _hs_tail) : NULL); \ + } \ + _hs_tail = _hs_e; \ + } \ + _hs_p = _hs_q; \ + } \ + if (_hs_tail != NULL) { \ + _hs_tail->next = NULL; \ + } \ + if (_hs_nmerges <= 1U) { \ + _hs_looping = 0; \ + (head)->hh.tbl->tail = _hs_tail; \ + DECLTYPE_ASSIGN(head, ELMT_FROM_HH((head)->hh.tbl, _hs_list)); \ + } \ + _hs_insize *= 2U; \ + } \ + HASH_FSCK(hh, head, "HASH_SRT"); \ + } \ +} while (0) + +/* This function selects items from one hash into another hash. + * The end result is that the selected items have dual presence + * in both hashes. There is no copy of the items made; rather + * they are added into the new hash through a secondary hash + * hash handle that must be present in the structure. */ +#define HASH_SELECT(hh_dst, dst, hh_src, src, cond) \ +do { \ + unsigned _src_bkt, _dst_bkt; \ + void *_last_elt = NULL, *_elt; \ + UT_hash_handle *_src_hh, *_dst_hh, *_last_elt_hh=NULL; \ + ptrdiff_t _dst_hho = ((char*)(&(dst)->hh_dst) - (char*)(dst)); \ + if ((src) != NULL) { \ + for (_src_bkt=0; _src_bkt < (src)->hh_src.tbl->num_buckets; _src_bkt++) { \ + for (_src_hh = (src)->hh_src.tbl->buckets[_src_bkt].hh_head; \ + _src_hh != NULL; \ + _src_hh = _src_hh->hh_next) { \ + _elt = ELMT_FROM_HH((src)->hh_src.tbl, _src_hh); \ + if (cond(_elt)) { \ + IF_HASH_NONFATAL_OOM( int _hs_oomed = 0; ) \ + _dst_hh = (UT_hash_handle*)(((char*)_elt) + _dst_hho); \ + _dst_hh->key = _src_hh->key; \ + _dst_hh->keylen = _src_hh->keylen; \ + _dst_hh->hashv = _src_hh->hashv; \ + _dst_hh->prev = _last_elt; \ + _dst_hh->next = NULL; \ + if (_last_elt_hh != NULL) { \ + _last_elt_hh->next = _elt; \ + } \ + if ((dst) == NULL) { \ + DECLTYPE_ASSIGN(dst, _elt); \ + HASH_MAKE_TABLE(hh_dst, dst, _hs_oomed); \ + IF_HASH_NONFATAL_OOM( \ + if (_hs_oomed) { \ + uthash_nonfatal_oom(_elt); \ + (dst) = NULL; \ + continue; \ + } \ + ) \ + } else { \ + _dst_hh->tbl = (dst)->hh_dst.tbl; \ + } \ + HASH_TO_BKT(_dst_hh->hashv, _dst_hh->tbl->num_buckets, _dst_bkt); \ + HASH_ADD_TO_BKT(_dst_hh->tbl->buckets[_dst_bkt], hh_dst, _dst_hh, _hs_oomed); \ + (dst)->hh_dst.tbl->num_items++; \ + IF_HASH_NONFATAL_OOM( \ + if (_hs_oomed) { \ + HASH_ROLLBACK_BKT(hh_dst, dst, _dst_hh); \ + HASH_DELETE_HH(hh_dst, dst, _dst_hh); \ + _dst_hh->tbl = NULL; \ + uthash_nonfatal_oom(_elt); \ + continue; \ + } \ + ) \ + HASH_BLOOM_ADD(_dst_hh->tbl, _dst_hh->hashv); \ + _last_elt = _elt; \ + _last_elt_hh = _dst_hh; \ + } \ + } \ + } \ + } \ + HASH_FSCK(hh_dst, dst, "HASH_SELECT"); \ +} while (0) + +#define HASH_CLEAR(hh,head) \ +do { \ + if ((head) != NULL) { \ + HASH_BLOOM_FREE((head)->hh.tbl); \ + uthash_free((head)->hh.tbl->buckets, \ + (head)->hh.tbl->num_buckets*sizeof(struct UT_hash_bucket)); \ + uthash_free((head)->hh.tbl, sizeof(UT_hash_table)); \ + (head) = NULL; \ + } \ +} while (0) + +#define HASH_OVERHEAD(hh,head) \ + (((head) != NULL) ? ( \ + (size_t)(((head)->hh.tbl->num_items * sizeof(UT_hash_handle)) + \ + ((head)->hh.tbl->num_buckets * sizeof(UT_hash_bucket)) + \ + sizeof(UT_hash_table) + \ + (HASH_BLOOM_BYTELEN))) : 0U) + +#ifdef NO_DECLTYPE +#define HASH_ITER(hh,head,el,tmp) \ +for(((el)=(head)), ((*(char**)(&(tmp)))=(char*)((head!=NULL)?(head)->hh.next:NULL)); \ + (el) != NULL; ((el)=(tmp)), ((*(char**)(&(tmp)))=(char*)((tmp!=NULL)?(tmp)->hh.next:NULL))) +#else +#define HASH_ITER(hh,head,el,tmp) \ +for(((el)=(head)), ((tmp)=DECLTYPE(el)((head!=NULL)?(head)->hh.next:NULL)); \ + (el) != NULL; ((el)=(tmp)), ((tmp)=DECLTYPE(el)((tmp!=NULL)?(tmp)->hh.next:NULL))) +#endif + +/* obtain a count of items in the hash */ +#define HASH_COUNT(head) HASH_CNT(hh,head) +#define HASH_CNT(hh,head) ((head != NULL)?((head)->hh.tbl->num_items):0U) + +typedef struct UT_hash_bucket { + struct UT_hash_handle *hh_head; + unsigned count; + + /* expand_mult is normally set to 0. In this situation, the max chain length + * threshold is enforced at its default value, HASH_BKT_CAPACITY_THRESH. (If + * the bucket's chain exceeds this length, bucket expansion is triggered). + * However, setting expand_mult to a non-zero value delays bucket expansion + * (that would be triggered by additions to this particular bucket) + * until its chain length reaches a *multiple* of HASH_BKT_CAPACITY_THRESH. + * (The multiplier is simply expand_mult+1). The whole idea of this + * multiplier is to reduce bucket expansions, since they are expensive, in + * situations where we know that a particular bucket tends to be overused. + * It is better to let its chain length grow to a longer yet-still-bounded + * value, than to do an O(n) bucket expansion too often. + */ + unsigned expand_mult; + +} UT_hash_bucket; + +/* random signature used only to find hash tables in external analysis */ +#define HASH_SIGNATURE 0xa0111fe1u +#define HASH_BLOOM_SIGNATURE 0xb12220f2u + +typedef struct UT_hash_table { + UT_hash_bucket *buckets; + unsigned num_buckets, log2_num_buckets; + unsigned num_items; + struct UT_hash_handle *tail; /* tail hh in app order, for fast append */ + ptrdiff_t hho; /* hash handle offset (byte pos of hash handle in element */ + + /* in an ideal situation (all buckets used equally), no bucket would have + * more than ceil(#items/#buckets) items. that's the ideal chain length. */ + unsigned ideal_chain_maxlen; + + /* nonideal_items is the number of items in the hash whose chain position + * exceeds the ideal chain maxlen. these items pay the penalty for an uneven + * hash distribution; reaching them in a chain traversal takes >ideal steps */ + unsigned nonideal_items; + + /* ineffective expands occur when a bucket doubling was performed, but + * afterward, more than half the items in the hash had nonideal chain + * positions. If this happens on two consecutive expansions we inhibit any + * further expansion, as it's not helping; this happens when the hash + * function isn't a good fit for the key domain. When expansion is inhibited + * the hash will still work, albeit no longer in constant time. */ + unsigned ineff_expands, noexpand; + + uint32_t signature; /* used only to find hash tables in external analysis */ +#ifdef HASH_BLOOM + uint32_t bloom_sig; /* used only to test bloom exists in external analysis */ + uint8_t *bloom_bv; + uint8_t bloom_nbits; +#endif + +} UT_hash_table; + +typedef struct UT_hash_handle { + struct UT_hash_table *tbl; + void *prev; /* prev element in app order */ + void *next; /* next element in app order */ + struct UT_hash_handle *hh_prev; /* previous hh in bucket order */ + struct UT_hash_handle *hh_next; /* next hh in bucket order */ + void *key; /* ptr to enclosing struct's key */ + unsigned keylen; /* enclosing struct's key len */ + unsigned hashv; /* result of hash-fcn(key) */ +} UT_hash_handle; + +#endif /* UTHASH_H */