diff options
| author |   Angelos Mouzakitis <a.mouzakitis@virtualopensystems.com> | 2023-10-10 14:33:42 +0000 |
|---|---|---|
| committer | Angelos Mouzakitis <a.mouzakitis@virtualopensystems.com> | 2023-10-10 14:33:42 +0000 |
| commit | af1a266670d040d2f4083ff309d732d648afba2a (patch) | |
| tree | 2fc46203448ddcc6f81546d379abfaeb323575e9 /roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research | |
| parent | e02cda008591317b1625707ff8e115a4841aa889 (diff) | |
Change-Id: Iaf8d18082d3991dec7c0ebbea540f092188eb4ec
Diffstat (limited to 'roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research')
58 files changed, 10903 insertions, 0 deletions
diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/BUILD b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/BUILD new file mode 100755 index 000000000..7b7d81b83 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/BUILD @@ -0,0 +1,44 @@ +# Description: brotli research tools. + +package(default_visibility = ["//visibility:public"]) + +licenses(["notice"]) # MIT + +cc_library( + name = "dm", + srcs = ["deorummolae.cc"], + hdrs = [ + "deorummolae.h", + "esaxx/sais.hxx", + ], +) + +cc_library( + name = "durchschlag", + srcs = ["durchschlag.cc"], + hdrs = ["durchschlag.h"], + deps = ["@divsufsort//:libdivsufsort"], +) + +cc_library( + name = "sieve", + srcs = ["sieve.cc"], + hdrs = ["sieve.h"], +) + +cc_binary( + name = "dictionary_generator", + srcs = ["dictionary_generator.cc"], + deps = [ + ":dm", + ":durchschlag", + ":sieve", + ], +) + +cc_binary( + name = "brotli_decoder", + srcs = ["brotli_decoder.c"], + linkstatic = 1, + deps = ["@org_brotli//:brotlidec"], +) diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/BUILD.libdivsufsort b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/BUILD.libdivsufsort new file mode 100644 index 000000000..ce60e9c20 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/BUILD.libdivsufsort @@ -0,0 +1,55 @@ +package( + default_visibility = ["//visibility:public"], +) + +cc_library( + name = "libdivsufsort", + srcs = [ + "lib/divsufsort.c", + "lib/sssort.c", + "lib/trsort.c", + "lib/utils.c", + ], + hdrs = [ + "include/config.h", + "include/divsufsort.h", + "include/divsufsort_private.h", + ], + copts = [ + "-DHAVE_CONFIG_H=1", + ], + includes = ["include"], +) + +commom_awk_replaces = ( + "gsub(/#cmakedefine/, \"#define\"); " + + "gsub(/@DIVSUFSORT_EXPORT@/, \"\"); " + + "gsub(/@DIVSUFSORT_IMPORT@/, \"\"); " + + "gsub(/@INLINE@/, \"inline\"); " + + "gsub(/@INCFILE@/, \"#include <inttypes.h>\"); " + + "gsub(/@SAUCHAR_TYPE@/, \"uint8_t\"); " + + "gsub(/@SAINT32_TYPE@/, \"int32_t\"); " + + "gsub(/@SAINT_PRId@/, \"PRId32\"); " +) + +genrule( + name = "config_h", + srcs = ["include/config.h.cmake"], + outs = ["include/config.h"], + cmd = ("awk '{ " + + "gsub(/@HAVE_IO_H 1@/, \"HAVE_IO_H 0\"); " + + commom_awk_replaces + + "print; }' $(<) > $(@)"), +) + +genrule( + name = "divsufsort_h", + srcs = ["include/divsufsort.h.cmake"], + outs = ["include/divsufsort.h"], + cmd = ("awk '{ " + + "gsub(/@W64BIT@/, \"\"); " + + "gsub(/@SAINDEX_TYPE@/, \"int32_t\"); " + + "gsub(/@SAINDEX_PRId@/, \"PRId32\"); " + + commom_awk_replaces + + "print; }' $(<) > $(@)"), +) diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/Makefile b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/Makefile new file mode 100644 index 000000000..751b3162c --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/Makefile @@ -0,0 +1,17 @@ +CC = g++ +CFLAGS += -O2 +CPPFLAGS += -std=c++11 +SOURCES = $(wildcard *.cc) +EXECUTABLES = $(SOURCES:.cc=) +BINDIR = bin + +all: $(EXECUTABLES) + +$(BINDIR): + mkdir -p $@ + +$(EXECUTABLES): $(BINDIR) + $(CC) $(CFLAGS) $(CPPFLAGS) $(addsuffix .cc, $@) -o $(BINDIR)/$@ -lgflags_nothreads + +clean: + rm -rf $(BINDIR) diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/README.md b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/README.md new file mode 100644 index 000000000..9c87ef820 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/README.md @@ -0,0 +1,67 @@ +## Introduction + +In this directory we publish simple tools to analyze backward reference distance distributions in LZ77 compression. We developed these tools to be able to make more efficient encoding of distances in large-window brotli. In large-window compression the average cost of a backward reference distance is higher, and this may allow for more advanced encoding strategies, such as delta coding or an increase in context size, to bring significant compression density improvements. Our tools visualize the backward references as histogram images, i.e., one pixel in the image shows how many distances of a certain range exist at a certain locality in the data. The human visual system is excellent at pattern detection, so we tried to roughly identify patterns visually before going into more quantitative analysis. These tools can turn out to be useful in development of other LZ77-based compressors and we hope you try them out. + + +## Tools +### find\_opt\_references + +This tool generates optimal (match-length-wise) backward references for every position in the input files and stores them in `*.dist` file described below. + +Example usage: + + find_opt_references input.txt output.dist + +### draw\_histogram + +This tool generates a visualization of the distribution of backward references stored in `*.dist` file. The original file size has to be specified as a second parameter. The output is a grayscale PGM (binary) image. + +Example usage: + + draw_histogram input.dist 65536 output.pgm + +Here's an example of resulting image: + + + +### draw\_diff + +This tool generates a diff PPM (binary) image between two input 8-bit PGM (binary) images. Input images must be of same size. Useful for comparing different backward references distributions for same input file. Normally used for comparison of output images from `draw_histogram` tool. + +Example usage: + + draw_diff image1.pgm image2.pgm diff.ppm + +For example the diff of this image + + + +and this image + + + +looks like this: + + + + +## Backward distance file format + +The format of `*.dist` files is as follows: + + [[ 0| match length][ 1|position|distance]...] + [1 byte| 4 bytes][1 byte| 4 bytes| 4 bytes] + +More verbose explanation: for each backward reference there is a position-distance pair, also a copy length may be specified. Copy length is prefixed with flag byte 0, position-distance pair is prefixed with flag byte 1. Each number is a 32-bit integer. Copy length always comes before position-distance pair. Standalone copy length is allowed, in this case it is ignored. + +Here's an example of how to read from `*.dist` file: + +```c++ +#include "read_dist.h" + +FILE* f; +int copy, pos, dist; +while (ReadBackwardReference(fin, ©, &pos, &dist)) { + ... +} +``` diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/WORKSPACE b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/WORKSPACE new file mode 100644 index 000000000..bb0f8ca08 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/WORKSPACE @@ -0,0 +1,12 @@ +workspace(name = "org_brotli_research") + +local_repository( + name = "org_brotli", + path = "..", +) + +new_local_repository( + name = "divsufsort", + build_file = "BUILD.libdivsufsort", + path = "libdivsufsort", +) diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/brotli_decoder.c b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/brotli_decoder.c new file mode 100644 index 000000000..b1d556d23 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/brotli_decoder.c @@ -0,0 +1,93 @@ +/* Copyright 2018 Google Inc. All Rights Reserved. + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> + +#include <brotli/decode.h> + +#define BUFFER_SIZE (1u << 20) + +typedef struct Context { + FILE* fin; + FILE* fout; + uint8_t* input_buffer; + uint8_t* output_buffer; + BrotliDecoderState* decoder; +} Context; + +void init(Context* ctx) { + ctx->fin = 0; + ctx->fout = 0; + ctx->input_buffer = 0; + ctx->output_buffer = 0; + ctx->decoder = 0; +} + +void cleanup(Context* ctx) { + if (ctx->decoder) BrotliDecoderDestroyInstance(ctx->decoder); + if (ctx->output_buffer) free(ctx->output_buffer); + if (ctx->input_buffer) free(ctx->input_buffer); + if (ctx->fout) fclose(ctx->fout); + if (ctx->fin) fclose(ctx->fin); +} + +void fail(Context* ctx, const char* message) { + fprintf(stderr, "%s\n", message); + exit(1); +} + +int main(int argc, char** argv) { + Context ctx; + BrotliDecoderResult result = BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT; + size_t available_in; + const uint8_t* next_in; + size_t available_out = BUFFER_SIZE; + uint8_t* next_out; + init(&ctx); + + ctx.fin = fdopen(STDIN_FILENO, "rb"); + if (!ctx.fin) fail(&ctx, "can't open input file"); + ctx.fout = fdopen(STDOUT_FILENO, "wb"); + if (!ctx.fout) fail(&ctx, "can't open output file"); + ctx.input_buffer = (uint8_t*)malloc(BUFFER_SIZE); + if (!ctx.input_buffer) fail(&ctx, "out of memory / input buffer"); + ctx.output_buffer = (uint8_t*)malloc(BUFFER_SIZE); + if (!ctx.output_buffer) fail(&ctx, "out of memory / output buffer"); + ctx.decoder = BrotliDecoderCreateInstance(0, 0, 0); + if (!ctx.decoder) fail(&ctx, "out of memory / decoder"); + BrotliDecoderSetParameter(ctx.decoder, BROTLI_DECODER_PARAM_LARGE_WINDOW, 1); + + next_out = ctx.output_buffer; + while (1) { + if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) { + if (feof(ctx.fin)) break; + available_in = fread(ctx.input_buffer, 1, BUFFER_SIZE, ctx.fin); + next_in = ctx.input_buffer; + if (ferror(ctx.fin)) break; + } else if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) { + fwrite(ctx.output_buffer, 1, BUFFER_SIZE, ctx.fout); + if (ferror(ctx.fout)) break; + available_out = BUFFER_SIZE; + next_out = ctx.output_buffer; + } else { + break; + } + result = BrotliDecoderDecompressStream( + ctx.decoder, &available_in, &next_in, &available_out, &next_out, 0); + } + if (next_out != ctx.output_buffer) { + fwrite(ctx.output_buffer, 1, next_out - ctx.output_buffer, ctx.fout); + } + if ((result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) || ferror(ctx.fout)) { + fail(&ctx, "failed to write output"); + } else if (result != BROTLI_DECODER_RESULT_SUCCESS) { + fail(&ctx, "corrupt input"); + } + cleanup(&ctx); + return 0; +} diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/brotlidump.py b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/brotlidump.py new file mode 100644 index 000000000..701893456 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/brotlidump.py @@ -0,0 +1,2362 @@ +#!python3 +"""Program to dump contents of Brotli compressed files showing the compression format. +Jurjen N.E. Bos, 2016. +I found the following issues with the Brotli format: +- The distance alphabet has size 16+(48<<POSTFIX), + but the last symbols are useless. + It could be lowered to 16+(44-POSTFIX<<POSTFIX), and this could matter. +- The block type code is useless if NBLTYPES==2, you would only need 1 symbol + anyway, so why don't you just switch to "the other" type? +""" +import struct +from operator import itemgetter, methodcaller +from itertools import accumulate, repeat +from collections import defaultdict, deque +from functools import partial + +class InvalidStream(Exception): pass +#lookup table +L, I, D = "literal", "insert©", "distance" +pL, pI, pD = 'P'+L, 'P'+I, 'P'+D + +def outputCharFormatter(c): + """Show character in readable format + """ + #TODO 2: allow hex only output + if 32<c<127: return chr(c) + elif c==10: return '\\n' + elif c==13: return '\\r' + elif c==32: return '" "' + else: return '\\x{:02x}'.format(c) + +def outputFormatter(s): + """Show string or char. + """ + result = '' + def formatSubString(s): + for c in s: + if c==32: yield ' ' + else: yield outputCharFormatter(c) + if len(result)<200: return ''.join(formatSubString(s)) + else: + return ''.join(formatSubString(s[:100]))+'...'+ \ + ''.join(formatSubString(s[-100:])) + + +class BitStream: + """Represent a bytes object. Can read bits and prefix codes the way + Brotli does. + """ + def __init__(self, byteString): + self.data = byteString + #position in bits: byte pos is pos>>3, bit pos is pos&7 + self.pos = 0 + + def __repr__(self): + """Representation + >>> olleke + BitStream(pos=0:0) + """ + return "BitStream(pos={:x}:{})".format(self.pos>>3, self.pos&7) + + def read(self, n): + """Read n bits from the stream and return as an integer. + Produces zero bits beyond the stream. + >>> olleke.data[0]==27 + True + >>> olleke.read(5) + 27 + + >>> olleke + BitStream(pos=0:5) + """ + value = self.peek(n) + self.pos += n + if self.pos>len(self.data)*8: + raise ValueError('Read past end of stream') + return value + + def peek(self, n): + """Peek an n bit integer from the stream without updating the pointer. + It is not an error to read beyond the end of the stream. + >>> olleke.data[:2]==b'\x1b\x2e' and 0x2e1b==11803 + True + >>> olleke.peek(15) + 11803 + >>> hex(olleke.peek(32)) + '0x2e1b' + """ + #read bytes that contain the data: self.data[self.pos>>3:self.pos+n+7>>3] + #convert to int: int.from_bytes(..., 'little') + #shift out the bits from the first byte: >>(self.pos&7) + #mask unwanted bits: & (1<<n)-1 + return int.from_bytes( + self.data[self.pos>>3:self.pos+n+7>>3], + 'little')>>(self.pos&7) & (1<<n)-1 + + def readBytes(self, n): + """Read n bytes from the stream on a byte boundary. + """ + if self.pos&7: raise ValueError('readBytes: need byte boundary') + result = self.data[self.pos>>3:(self.pos>>3)+n] + self.pos += 8*n + return result + +#-----------------------Symbol------------------------------------------- +class Symbol: + """A symbol in a code. + Refers back to the code that contains it. + Index is the place in the alphabet of the symbol. + """ + __slots__ = 'code', 'index' + def __init__(self, code, value): + self.code = code + self.index = value + + def __repr__(self): + return 'Symbol({}, {})'.format(self.code.name, self.index) + + def __len__(self): + """Number of bits in the prefix notation of this symbol + """ + return self.code.length(self.index) + + def __int__(self): + return self.index + + #these routines call equivalent routine in Code class + def bitPattern(self): + """Value of the symbol in the stream + """ + return self.code.bitPattern(self.index) + + def extraBits(self): + """Number of extra bits to read for this symbol + """ + return self.code.extraBits(self.index) + + def __str__(self): + """Short descriptor of the symbol without extra bits. + """ + return self.code.mnemonic(self.index) + + #requiring optional extra bits, if self.code supports them + def value(self, extra=None): + """The value used for processing. Can be a tuple. + with optional extra bits + """ + if isinstance(self.code, WithExtra): + if not 0<=extra<1<<self.extraBits(): + raise ValueError("value: extra value doesn't fit in extraBits") + return self.code.value(self.index, extra) + if extra is not None: + raise ValueError('value: no extra bits for this code') + return self.code.value(self.index) + + def explanation(self, extra=None): + """Long explanation of the value from the numeric value + with optional extra bits + Used by Layout.verboseRead when printing the value + """ + if isinstance(self.code, WithExtra): + return self.code.callback(self, extra) + return self.code.callback(self) + +#========================Code definitions================================== +class RangeDecoder: + """A decoder for the Code class that assumes the symbols + are encoded consecutively in binary. + It all depends on the "alphabetSize" property. + The range runs from 0 to alphabetSize-1. + This is the default decoder. + """ + def __init__(self, *, alphabetSize=None, bitLength=None, **args): + if bitLength is not None: alphabetSize = 1<<bitLength + if alphabetSize is not None: + self.alphabetSize = alphabetSize + self.maxLength = (alphabetSize-1).bit_length() + + def __len__(self): + return self.alphabetSize + + def __iter__(self): + """Produce all symbols. + """ + return map(partial(Symbol, self), range(len(self))) + + def __getitem__(self, index): + if index>=self.alphabetSize: raise ValueError('index out of range') + return Symbol(self, index) + + def bitPattern(self, index): + return '{:0{}b}'.format(index, self.maxLength) + + def length(self, index): + """Encoding length of given symbol. + Does not depend on index in this case. + """ + return self.maxLength + + def decodePeek(self, data): + """Find which symbol index matches the given data (from peek, as a number) + and return the number of bits decoded. + Can also be used to figure out length of a symbol. + """ + return self.maxLength, Symbol(self, data&(1<<self.maxLength)-1) + +class PrefixDecoder: + """A decoder for the Code class that uses a prefix code. + The code is determined by encoding: + encode[p] gives the index corresponding to bit pattern p. + Used setDecode(decodeTable) to switch the decoder from the default + to a prefix decoder, or pass decodeTable at init. + You can also use setLength(lengthTable) + to define the encoding from the lengths. + The set of symbol values does not need to be consecutive. + """ + def __init__(self, *, decodeTable=None, **args): + if decodeTable is not None: self.setDecode(decodeTable) + + def __len__(self): + return len(self.decodeTable) + + def __iter__(self): + def revBits(index): + return self.bitPattern(index)[::-1] + return ( + Symbol(self, index) + for index in sorted(self.decodeTable.values(), key=revBits) + ) + + def __getitem__(self, index): + if index not in self.lengthTable: + raise ValueError('No symbol {}[{}]'.format( + self.__class__.__name__, index)) + return Symbol(self, index) + + def bitPattern(self, index): + bits = next(b for (b,s) in self.decodeTable.items() if s==index) + return '{:0{}b}'.format(bits, self.length(index)) + + def length(self, index): + """Encoding length of given symbol. + """ + return self.lengthTable[index] + + def decodePeek(self, data): + """Find which symbol index matches the given data (from peek, as a number) + and return the number of bits decoded. + Can also be used to figure out length of a symbol. + """ + #do binary search for word length + #invariant: lo<=length<=hi + lo, hi = self.minLength, self.maxLength + while lo<=hi: + mid = lo+hi>>1 + #note lo<=mid<hi at this point + mask = (1<<mid)-1 + #lets see what happens if we guess length is mid + try: index = self.decodeTable[data&mask] + except KeyError: + #too many bits specified, reduce estimated length + hi = mid-1 + continue + #we found a symbol, but there could be a longer match + symbolLength = self.lengthTable[index] + if symbolLength<=mid: + #all bits match, symbol must be right + return symbolLength, Symbol(self, index) + #there must be more bits to match + lo = mid+1 + return lo, Symbol(self, index) + + #routine to set up the tables + def setDecode(self, decodeTable): + """Store decodeTable, + and compute lengthTable, minLength, maxLength from encodings. + """ + self.decodeTable = decodeTable + #set of symbols with unknown length + todo = set(decodeTable) + #bit size under investigation + maskLength = 0 + lengthTable = {} + while todo: + mask = (1<<maskLength)-1 + #split the encodings that we didn't find yet using b bits + splitSymbols = defaultdict(list) + for s in todo: splitSymbols[s&mask].append(s) + #unique encodings have a length of maskLength bits + #set length, and remove from todo list + for s,subset in splitSymbols.items(): + if len(subset)==1: + lengthTable[self.decodeTable[s]] = maskLength + todo.remove(s) + #now investigate with longer mask + maskLength +=1 + #save result + self.lengthTable = lengthTable + self.minLength = min(lengthTable.values()) + self.maxLength = max(lengthTable.values()) + self.switchToPrefix() + + def setLength(self, lengthTable): + """Given the bit pattern lengths for symbols given in lengthTable, + set decodeTable, minLength, maxLength + """ + self.lengthTable = lengthTable + self.minLength = min(lengthTable.values()) + self.maxLength = max(lengthTable.values()) + #compute the backwards codes first; then reverse them + #compute (backwards) first code for every separate lengths + nextCodes = [] + #build codes for each length, from right to left + code = 0 + for bits in range(self.maxLength+1): + code <<= 1 + nextCodes.append(code) + code += sum(x==bits for x in lengthTable.values()) + self.decodeTable = {} + #count codes for each length, and store reversed in the table + for symbol in sorted(lengthTable): + bits = lengthTable[symbol] + bitpattern = '{:0{}b}'.format(nextCodes[bits], bits) + self.decodeTable[int(bitpattern[::-1], 2)] = symbol + nextCodes[bits] += 1 + self.switchToPrefix() + + def switchToPrefix(self): + """This routine makes sure the prefix decoder is activated. + """ + self.mode = PrefixDecoder + +class Code(RangeDecoder, PrefixDecoder): + """An alphabet of symbols, that can be read from a stream. + If you use setDecode or setLength, you have a prefix code, + otherwise you have a range code. + Features: + code[index] produces symbol with given index + value(index): value of symbol + mnemonic(index): short description of symbol + explanation(index): show meaning of symbol, shown in Layout.verboseRead + iter(code): produce all symbols in some order + name: show as context in Layout.verboseRead + """ + name = '?' + #callback is a function that gets the symbol and the extra bits + #default callback calls explanation + def __init__(self, name=None, *, callback=None, description='', **args): + """Don't forget to set either alphabetSize or decodeTable + """ + #set name when provided, otherwise take class variable + if name is not None: self.name = name + if callback is not None: self.callback = callback + self.description = description + #mode switch + if 'bitLength' in args or 'alphabetSize' in args: + self.mode = RangeDecoder + RangeDecoder.__init__(self, **args) + elif 'decodeTable' in args: + self.mode = PrefixDecoder + PrefixDecoder.__init__(self, **args) + else: + super().__init__(**args) + + def __repr__(self): + return self.__class__.__name__+' '+self.name + + #the routines that get switched between RangeDecoder and PrefixDecoder + def __len__(self): return self.mode.__len__(self) + def __iter__(self): return self.mode.__iter__(self) + def __getitem__(self, index): return self.mode.__getitem__(self, index) + def bitPattern(self, index): return self.mode.bitPattern(self, index) + def length(self, index): return self.mode.length(self, index) + def decodePeek(self, data): return self.mode.decodePeek(self, data) + #general routines + def value(self, index, extra=None): + """Get value of symbol for computations. + Override where needed. + """ + if extra is not None: + raise ValueError('value: no extra for this symbol') + return index + + def mnemonic(self, index): + """Give mnemonic of symbol. + Override where needed. + """ + return str(self.value(index)) + + def callback(self, symbol): + return self.explanation(symbol.index) + + def explanation(self, index): + """Long explanation of the value from the numeric value + This is a default routine. + You can customize in three ways: + - set description to add some text + - override to get more control + - set callback to make it dependent on you local variables + """ + value = self.value(index) + return '{0}{1}: {2}'.format( + self.description and self.description+': ', + self.bitPattern(index), + value, + ) + + def extraBits(self, index): + return 0 + + #Routines that use the decode interface + def showCode(self, width=80): + """Show all words of the code in a nice format. + """ + #make table of all symbols with binary strings + symbolStrings = [ + (self.bitPattern(s.index), self.mnemonic(s.index)) + for s in self + ] + #determine column widths the way Lisp programmers do it + leftColWidth, rightColWidth = map(max, map( + map, + repeat(len), + zip(*symbolStrings) + )) + colwidth = leftColWidth+rightColWidth + columns = 81//(colwidth+2) + rows = -(-len(symbolStrings)//columns) + def justify(bs): + b,s = bs + return b.rjust(leftColWidth)+':'+s.ljust(rightColWidth) + for i in range(rows): + print(' '.join(map(justify, symbolStrings[i::rows])).rstrip()) + + def readTuple(self, stream): + """Read symbol from stream. Returns symbol, length. + """ + length, symbol = self.decodePeek(stream.peek(self.maxLength)) + stream.pos += length + return length, symbol + + def readTupleAndExtra(self, stream): + return self.readTuple(stream)+(0, None) + +class WithExtra(Code): + """Extension for Code so that symbol may have extra bits associated. + If you supply an extraTable, you can use extraBits + You can define an extraTable, + which allows to call extraBits to get the number of extraBits. + Otherwise, you can supply extraBits yourself. + Routine readTupleAndExtra now reads the extra bits too. + Value probably needs to be overridden; see Enumerator. + Note: this does not give you an decodeTable. + """ + #redefine these if you don't want to use an extraTable + def extraBits(self, index): + """Get the number of extra bits for this symbol. + """ + return self.extraTable[index] + + def mnemonic(self, index): + """This value must be independent of extra. + """ + return str(index) + + def readTupleAndExtra(self, stream): + """Read symbol and extrabits from stream. + Returns symbol length, symbol, extraBits, extra + >>> olleke.pos = 6 + >>> MetablockLengthAlphabet().readTupleAndExtra(olleke) + (2, Symbol(MLEN, 4), 16, 46) + """ + length, symbol = self.decodePeek(stream.peek(self.maxLength)) + stream.pos += length + extraBits = self.extraBits(symbol.index) + return length, symbol, extraBits, stream.read(extraBits) + + def explanation(self, index, extra=None): + """Expanded version of Code.explanation supporting extra bits. + If you don't supply extra, it is not mentioned. + """ + extraBits = 0 if extra is None else self.extraBits(index) + if not hasattr(self, 'extraTable'): + formatString = '{0}{3}' + lo = hi = value = self.value(index, extra) + elif extraBits==0: + formatString = '{0}{2}: {3}' + lo, hi = self.span(index) + value = lo + else: + formatString = '{0}{1} {2}: {3}-{4}; {3}+{5}={6}' + lo, hi = self.span(index) + value = lo+extra + return formatString.format( + self.description and self.description+': ', + 'x'*extraBits, + self.bitPattern(index), + lo, hi, + extra, + value, + ) + + def callback(self, symbol, extra): + return self.explanation(symbol.index, extra) + +class BoolCode(Code): + """Same as Code(bitLength=1), but shows a boolean. + """ + def __init__(self, name=None, **args): + super().__init__(name, bitLength=1, **args) + + def value(self, index, extra=None): + return bool(super().value(index, extra)) + +class Enumerator(WithExtra): + """Code that is defined by the ExtraTable. + extraTable is a class variable that contains + the extraBits of the symbols from 0 + value0 contains the value of symbol 0 + encodings is not neccessary, but allowed. + Note: place for FixedCode to make sure extraBits works + """ + def __init__(self, name=None, **args): + #if there is no decodeTable to determine length, compute it ourselves + if 'decodeTable' not in args: + args['alphabetSize'] = len(self.extraTable) + super().__init__(name, **args) + + def __len__(self): + return len(self.extraTable) + + def __getitem__(self, index): + """Faster than PrefixDecoder + """ + if index>=len(self.extraTable): + raise ValueError("No symbol {}[{}]".format( + self.__class__.__name__, index)) + return Symbol(self, index) + + def value(self, index, extra): + """Override if you don't define value0 and extraTable + """ + lower, upper = self.span(index) + value = lower+(extra or 0) + if value>upper: + raise ValueError('value: extra out of range') + return value + + def span(self, index): + """Give the range of possible values in a tuple + Useful for mnemonic and explanation + """ + lower = self.value0+sum(1<<x for x in self.extraTable[:index]) + upper = lower+(1<<self.extraTable[index]) + return lower, upper-1 + +#======================Code subclasses====================================== +#Alphabets used in the metablock header---------------------------------- +#For prefix codes +class PrefixCodeHeader(WithExtra): + """Header of prefix codes. + """ + def __init__(self, codename): + super().__init__('PFX', bitLength=2) + #this is the name of the code that it describes + self.codename = codename + + def extraBits(self, index): + return 2 if index==1 else 0 + + def value(self, index, extra): + """Returns ('Simple', #codewords) or ('Complex', HSKIP) + """ + if index==1: + if extra>3: + raise ValueError('value: extra out of range') + return 'Simple', extra+1 + if extra: + raise ValueError('value: extra out of range') + return 'Complex', index + + def explanation(self, index, extra): + if index==1: + return '{} is simple with {} code word{}'.format( + self.codename, extra+1, 's' if extra else '') + lengths = [1, 2, 3, 4, 0, 5, 17, 6] + return '{} is complex with lengths {}...'.format( + self.codename, + ','.join( + map(str, lengths[index:index+5])) + ) + +class TreeShapeAlhabet(BoolCode): + """The bit used to indicate if four word code is "deep" or "wide" + """ + name = 'SHAPE' + def value(self, index): + return [(2,2,2,2), (1,2,3,3)][index] + + def explanation(self, index): + return str(bool(index))+': lengths {},{},{},{}'.format(*self.value(index)) + +class LengthOfLengthAlphabet(Code): + """For use in decoding complex code descriptors. + >>> lengthOfLengthAlphabet = LengthOfLengthAlphabet('') + >>> print(lengthOfLengthAlphabet[2]) + coded with 2 bits + >>> len(lengthOfLengthAlphabet[0]) + 2 + >>> [len(lengthOfLengthAlphabet[x]) for x in range(6)] + [2, 4, 3, 2, 2, 4] + >>> lengthOfLengthAlphabet.showCode() + 00:skipped 01:coded with 4 bits 0111:coded with 1 bits + 10:coded with 3 bits 011:coded with 2 bits 1111:coded with 5 bits + """ + decodeTable = { + 0b00:0, 0b10:3, + 0b0111:1, 0b01:4, + 0b011:2, 0b1111:5, + } + + def __init__(self, name=None, **args): + super().__init__(name, decodeTable=self.decodeTable, **args) + + def mnemonic(self, index): + if index==0: return 'skipped' + return 'coded with {} bits'.format(index) + + def explanation(self, index, extra=None): + return self.description+': '+self.mnemonic(index) + +class LengthAlphabet(WithExtra): + """Length of symbols + Used during construction of a code. + """ + def __init__(self, name): + super().__init__(name, alphabetSize=18) + + def extraBits(self, index): + return {16:2, 17:3}.get(index, 0) + + def mnemonic(self, index): + if index==0: return 'unused' + elif index==16: return 'rep xx' + elif index==17: return 'zero xxx' + else: return 'len {}'.format(index) + + def explanation(self, index, extra): + return self.description.format(self[index], extra) + + def value(self, index, extra): + #the caller got the length already, so extra is enough + return extra + +#Stream header +class WindowSizeAlphabet(Code): + """The alphabet used for window size in the stream header. + >>> WindowSizeAlphabet()[10].explanation() + 'windowsize=(1<<10)-16=1008' + """ + decodeTable = { + 0b0100001: 10, 0b1100001: 14, 0b0011: 18, 0b1011: 22, + 0b0110001: 11, 0b1110001: 15, 0b0101: 19, 0b1101: 23, + 0b1000001: 12, 0b0: 16, 0b0111: 20, 0b1111: 24, + 0b1010001: 13, 0b0000001: 17, 0b1001: 21, + 0b0010001: None, + } + + name = 'WSIZE' + + def __init__(self, name=None): + super().__init__(name, decodeTable=self.decodeTable) + + def value(self, index): + #missing value gives index None + if index is None: return None + return (1<<index)-16 + + def explanation(self, index): + return 'windowsize=(1<<{})-16={}'.format( + index, (1<<index)-16) + +#Metablock +class MetablockLengthAlphabet(WithExtra): + """Used for the meta block length; + also indicates a block with no data + >>> metablockLengthAlphabet = MetablockLengthAlphabet() + >>> metablockLengthAlphabet[0]; str(metablockLengthAlphabet[0]) + Symbol(MLEN, 0) + 'empty' + >>> metablockLengthAlphabet[3] + Traceback (most recent call last): + ... + ValueError: No symbol MetablockLengthAlphabet[3] + >>> print(metablockLengthAlphabet[4]) + hhhh00 + >>> metablockLengthAlphabet[4].value(0x1000) + 4097 + >>> metablockLengthAlphabet[5].value(0x1000) + Traceback (most recent call last): + ... + InvalidStream: Zeros in high nibble of MLEN + >>> metablockLengthAlphabet[5].explanation(0x12345) + 'data length: 12345h+1=74566' + >>> metablockLengthAlphabet.showCode() + 00:hhhh00 10:hhhhhh10 01:hhhhh01 11:empty + """ + decodeTable = {0b11:0, 0b00:4, 0b01:5, 0b10:6} + + name = 'MLEN' + def __init__(self, name=None): + super().__init__(name, decodeTable=self.decodeTable) + + def extraBits(self, index): + return index*4 + + def mnemonic(self, index): + if index==0: return 'empty' + return 'h'*(self.extraBits(index)//4)+self.bitPattern(index) + + def value(self, index, extra): + extraBits = self.extraBits(index) + if not 0<=extra<1<<extraBits: + raise ValueError('value: extra out of range') + if index==0: return 0 + if index>4 and extra>>extraBits-4==0: raise InvalidStream( + 'Zeros in high nibble of MLEN') + return extra+1 + + def explanation(self, index, extra): + if index==0: return '11: empty block' + extraBits = self.extraBits(index) + return 'data length: {:0{}x}h+1={}'.format(extra, extraBits//4, extra+1) + + +class ReservedAlphabet(BoolCode): + """The reserved bit that must be zero. + """ + name = 'RSVD' + def value(self, index): + if index: raise ValueError('Reserved bit is not zero') + + def explanation(self, index): + return 'Reserved (must be zero)' + +class FillerAlphabet(Code): + def __init__(self, *, streamPos): + super().__init__('SKIP', bitLength=(-streamPos)&7) + + def explanation(self, index): + return '{} bit{} ignored'.format( + self.length(index), + '' if self.length(index)==1 else 's', + ) + +class SkipLengthAlphabet(WithExtra): + """Used for the skip length in an empty metablock + >>> skipLengthAlphabet = SkipLengthAlphabet() + >>> skipLengthAlphabet[0]; str(skipLengthAlphabet[0]) + Symbol(SKIP, 0) + 'empty' + >>> skipLengthAlphabet[4] + Traceback (most recent call last): + ... + ValueError: index out of range + >>> print(skipLengthAlphabet[3]) + hhhhhh11 + >>> skipLengthAlphabet[2].value(0x1000) + 4097 + >>> skipLengthAlphabet[3].value(0x1000) + Traceback (most recent call last): + ... + InvalidStream: Zeros in high byte of SKIPBYTES + >>> skipLengthAlphabet[3].explanation(0x12345) + 'skip length: 12345h+1=74566' + >>> skipLengthAlphabet.showCode() + 00:empty 01:hh01 10:hhhh10 11:hhhhhh11 + """ + def __init__(self): + super().__init__('SKIP', bitLength=2) + + def extraBits(self, index): + return index*8 + + def mnemonic(self, index): + if index==0: return 'empty' + return 'h'*(self.extraBits(index)//4)+self.bitPattern(index) + + def value(self, index, extra): + extraBits = self.extraBits(index) + if not 0<=extra<1<<extraBits: + raise ValueError('value: extra out of range') + if index==0: return 0 + if index>1 and extra>>extraBits-8==0: + raise InvalidStream('Zeros in high byte of SKIPBYTES') + return extra+1 + + def explanation(self, index, extra): + if index==0: return '00: no skip' + extraBits = self.extraBits(index) + return 'skip length: {:{}x}h+1={}'.format(extra, extraBits//8, extra+1) + + +class TypeCountAlphabet(Enumerator): + """Used for giving block type counts and tree counts. + >>> TypeCountAlphabet(description='').showCode() + 0:0 0101:xx,0101 1011:xxxxx,1011 + 0001:0001 1101:xxxxxx,1101 0111:xxx,0111 + 1001:xxxx,1001 0011:x,0011 1111:xxxxxxx,1111 + """ + decodeTable = { + 0b0: 0, 0b1001: 5, + 0b0001: 1, 0b1011: 6, + 0b0011: 2, 0b1101: 7, + 0b0101: 3, 0b1111: 8, + 0b0111: 4, + } + + value0 = 1 + extraTable = [0, 0, 1, 2, 3, 4, 5, 6, 7] + name = 'BT#' + + def __init__(self, name=None, *, description): + super().__init__( + name, + decodeTable=self.decodeTable, + description=description) + + def mnemonic(self, index): + if index==0: return '0' + if index==1: return '0001' + return 'x'*(self.extraBits(index))+','+self.bitPattern(index) + + def explanation(self, index, extra): + value = self.value(index, extra) + description = self.description + if value==1: description = description[:-1] + return '{}: {} {}'.format( + self.mnemonic(index), + value, + description) + +class BlockTypeAlphabet(Code): + """The block types; this code works for all three kinds. + >>> b = BlockTypeAlphabet('T', NBLTYPES=5) + >>> print(*(x for x in b)) + prev +1 #0 #1 #2 #3 #4 + """ + def __init__(self, name, NBLTYPES, **args): + super().__init__(name, alphabetSize=NBLTYPES+2, **args) + self.NBLTYPES = NBLTYPES + + def mnemonic(self, index): + if index==0: return 'prev' + elif index==1: return '+1' + else: return '#'+str(index-2) + + def value(self, index): + return index-2 + + def explanation(self, index): + if index==0: return '0: previous' + elif index==1: return '1: increment' + else: return 'Set block type to: '+str(index-2) + +class BlockCountAlphabet(Enumerator): + """Block counts + >>> b = BlockCountAlphabet('L') + >>> print(b[25]) + [24*x]: BC16625-16793840 + """ + + value0 = 1 + extraTable = [2,2,2,2,3, 3,3,3,4,4, 4,4,5,5,5, 5,6,6,7,8, 9,10,11,12,13, 24] + def __init__(self, name, **args): + super().__init__(name, alphabetSize=26, **args) + + def mnemonic(self, index): + extraBits = self.extraBits(index) + return '{}: BC{}-{}'.format( + 'x'*extraBits if index<5 else '[{}*x]'.format(extraBits), + *self.span(index)) + + def explanation(self, index, extra): + return 'Block count: '+super().explanation(index, extra) + +class DistanceParamAlphabet(WithExtra): + """The distance parameters NPOSTFIX and NDIRECT. + Although these are treated as two in the description, this is easier. + """ + def __init__(self): + super().__init__('DIST', bitLength=2) + + def extraBits(self, index): + return 4 + + def value(self, index, extra): + """Returns NPOSTFIX and NDIRECT<<NPOSTFIX + """ + if extra>15: + raise ValueError('value: extra out of range') + return index, extra<<index + + def explanation(self, index, extra): + return '{} postfix bits and {:04b}<<{}={} direct codes'.format( + index, extra, index, extra<<index) + + def mnemonic(self, index): + return 'PF'+str(index) + +class LiteralContextMode(Code): + """For the literal context modes. + >>> LiteralContextMode().showCode() + 00:LSB6 01:MSB6 10:UTF8 11:Signed + >>> LiteralContextMode().explanation(2) + 'Context mode for type 9: 2(UTF8)' + """ + + def __init__(self, *, number=9): + super().__init__('LC'+str(number), bitLength=2) + self.number = number + + def mnemonic(self, index): + return ['LSB6', 'MSB6', 'UTF8', 'Signed'][index] + + def explanation(self, index): + return 'Context mode for type {}: {}({})'.format( + self.number, + index, + self.mnemonic(index)) + +class RLEmaxAlphabet(Enumerator): + """Used for describing the run length encoding used for describing context maps. + >>> RLEmaxAlphabet().showCode() + 0:1 1:more + """ + value0 = 0 + extraTable = [0, 4] + name = 'RLE#' + + def mnemonic(self, index): + return ['1', 'more'][index] + + def explanation(self, index, extra): + description = self.description and self.description+': ' + if index==0: return description+'No RLE coding' + return '{}xxxx 1: RLEMAX={}'.format(description, extra+1) + +class TreeAlphabet(WithExtra): + """The alphabet to enumerate entries (called trees) in the context map. + parameters are RLEMAX and NTREES + >>> t = TreeAlphabet('', RLEMAX=3, NTREES=5) + >>> len(t) + 8 + >>> print(t[2]) + xx+4 zeroes + >>> t[3].explanation(2) + '8+010=10 zeroes' + >>> t[0].value(0) + (1, 0) + """ + name = 'CMI' + def __init__(self, name=None, *, RLEMAX, NTREES, **args): + super().__init__(name, alphabetSize=RLEMAX+NTREES, **args) + self.RLEMAX = RLEMAX + self.NTREES = NTREES + + def extraBits(self, index): + if 0<index<=self.RLEMAX: return index + return 0 + + def mnemonic(self, index): + if index==0: return 'map #0' + if index<=self.RLEMAX: + return '{}+{} zeroes'.format('x'*index, 1<<index) + return 'map #{}'.format(index-self.RLEMAX) + + def value(self, index, extra): + """Give count and value.""" + index = index + if index==0: return 1, 0 + if index<=self.RLEMAX: return (1<<index)+extra, 0 + return 1, index-self.RLEMAX + + def explanation(self, index, extra): + description = self.description and self.description+': ' + if index==0: return description+'map #0' + if index<=self.RLEMAX: + return '{}+{:0{}b}={} zeroes'.format( + (1<<index), + extra, self.extraBits(index), + (1<<index)+extra) + return '{}map #{}-{}={}'.format( + description, + index, self.RLEMAX, index-self.RLEMAX) + +#Prefix alphabets for the data stream---------------------------------- +class LiteralAlphabet(Code): + """Alphabet of symbols. + """ + minLength = maxLength = 8 + def __init__(self, number): + super().__init__('L'+str(number), alphabetSize=1<<8) + + def mnemonic(self, index): + return outputCharFormatter(index) + + def value(self, index, extra=None): + return index + + def explanation(self, index, extra=None): + return self.mnemonic(index) + +class InsertLengthAlphabet(Enumerator): + """Intern code for insert counts + """ + value0 = 0 + extraTable = [0,0,0,0,0, 0,1,1,2,2, 3,3,4,4,5, 5,6,7,8,9, 10,12,14,24] + +class CopyLengthAlphabet(Enumerator): + value0 = 2 + extraTable = [0,0,0,0,0, 0,0,0,1,1, 2,2,3,3,4, 4,5,5,6,7, 8,9,10,24] + +class InsertAndCopyAlphabet(WithExtra): + """The insert and copy code + >>> for x in range(0,704,704//13): + ... print('{:10b}'.format(x), InsertAndCopyAlphabet()[x]) + 0 I0C2&D=0 + 110110 I6+xC8&D=0 + 1101100 I5C22+xxx&D=0 + 10100010 I4C4 + 11011000 I3C10+x + 100001110 I14+xxC8 + 101000100 I10+xxC22+xxx + 101111010 I98+xxxxxC14+xx + 110110000 I6+xC70+xxxxx + 111100110 I1090+[10*x]C8 + 1000011100 I26+xxxC326+[8*x] + 1001010010 I322+[8*x]C14+xx + 1010001000 I194+[7*x]C70+xxxxx + 1010111110 I22594+[24*x]C1094+[10*x] + """ + insertLengthAlphabet = InsertLengthAlphabet(None) + copyLengthAlphabet = CopyLengthAlphabet(None) + + def __init__(self, number=''): + super().__init__('IC'+str(number), bitLength=10) + + def __len__(self): + return 704 + + def extraBits(self, index): + insertSymbol, copySymbol, dist0 = self.splitSymbol(index) + return InsertLengthAlphabet.extraTable[insertSymbol.index] + \ + CopyLengthAlphabet.extraTable[copySymbol.index] + + def splitSymbol(self, index): + """Give relevant values for computations: + (insertSymbol, copySymbol, dist0flag) + """ + #determine insert and copy upper bits from table + row = [0,0,1,1,2,2,1,3,2,3,3][index>>6] + col = [0,1,0,1,0,1,2,0,2,1,2][index>>6] + #determine inserts and copy sub codes + insertLengthCode = row<<3 | index>>3&7 + if row: insertLengthCode -= 8 + copyLengthCode = col<<3 | index&7 + return ( + Symbol(self.insertLengthAlphabet, insertLengthCode), + Symbol(self.copyLengthAlphabet, copyLengthCode), + row==0 + ) + + def mnemonic(self, index): + """Make a nice mnemonic + """ + i,c,d0 = self.splitSymbol(index) + iLower, _ = i.code.span(i.index) + iExtra = i.extraBits() + cLower, _ = c.code.span(c.index) + cExtra = c.extraBits() + return 'I{}{}{}C{}{}{}{}'.format( + iLower, + '+' if iExtra else '', + 'x'*iExtra if iExtra<6 else '[{}*x]'.format(iExtra), + cLower, + '+' if cExtra else '', + 'x'*cExtra if cExtra<6 else '[{}*x]'.format(cExtra), + '&D=0' if d0 else '') + + def value(self, index, extra): + i,c,d0 = self.splitSymbol(index) + iExtra = i.extraBits() + ce, ie = extra>>iExtra, extra&(1<<iExtra)-1 + insert = i.value(ie) + copy = c.value(ce) + return insert, copy, d0 + + def explanation(self, index, extra): + insert, copy, d0 = self.value(index, extra) + if d0: return 'Literal: {}, copy: {}, same distance'.format(insert, copy) + else: return 'Literal: {}, copy: {}'.format(insert, copy) + +class DistanceAlphabet(WithExtra): + """Represent the distance encoding. + Dynamically generated alphabet. + This is what the documentation should have said: + Ignoring offsets for the moment, the "long" encoding works as follows: + Write the distance in binary as follows: + 1xy..yz..z, then the distance symbol consists of n..nxz..z + Where: + n is one less than number of bits in y + x is a single bit + y..y are n+1 extra bits (encoded in the bit stream) + z..z is NPOSTFIX bits that are part of the symbol + The offsets are so as to start at the lowest useable value: + if 1xyyyyz = distance +(4<<POSTFIX)-NDIRECT-1 + then n..nxz..z is symbol -NDIRECT-16 + >>> d = DistanceAlphabet('D', NPOSTFIX=2, NDIRECT=10) + >>> print(d[4], d[17], d[34]) + last-1 1 10xx00-5 + >>> [str(d[x]) for x in range(26, 32)] + ['10x00-5', '10x01-5', '10x10-5', '10x11-5', '11x00-5', '11x01-5'] + """ + def __init__(self, number, *, NPOSTFIX, NDIRECT): + self.NPOSTFIX = NPOSTFIX + self.NDIRECT = NDIRECT + #set length + #Actually, not all symbols are used, + #only NDIRECT+16+(44-2*POSTFIX<<NPOSTFIX) + super().__init__('D'+str(number), + alphabetSize=self.NDIRECT+16+(48<<self.NPOSTFIX)) + + def extraBits(self, index): + """Indicate how many extra bits are needed to interpret symbol + >>> d = DistanceAlphabet('D', NPOSTFIX=2, NDIRECT=10) + >>> [d[i].extraBits() for i in range(26)] + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + >>> [d[i].extraBits() for i in range(26,36)] + [1, 1, 1, 1, 1, 1, 1, 1, 2, 2] + """ + if index<16+self.NDIRECT: return 0 + return 1 + ((index - self.NDIRECT - 16) >> (self.NPOSTFIX + 1)) + + def value(self, dcode, dextra): + """Decode value of symbol together with the extra bits. + >>> d = DistanceAlphabet('D', NPOSTFIX=2, NDIRECT=10) + >>> d[34].value(2) + (0, 35) + """ + if dcode<16: + return [(1,0),(2,0),(3,0),(4,0), + (1,-1),(1,+1),(1,-2),(1,+2),(1,-3),(1,+3), + (2,-1),(2,+1),(2,-2),(2,+2),(2,-3),(2,+3) + ][dcode] + if dcode<16+self.NDIRECT: + return (0,dcode-16) + #we use the original formulas, instead of my clear explanation + POSTFIX_MASK = (1 << self.NPOSTFIX) - 1 + ndistbits = 1 + ((dcode - self.NDIRECT - 16) >> (self.NPOSTFIX + 1)) + hcode = (dcode - self.NDIRECT - 16) >> self.NPOSTFIX + lcode = (dcode - self.NDIRECT - 16) & POSTFIX_MASK + offset = ((2 + (hcode & 1)) << ndistbits) - 4 + distance = ((offset + dextra) << self.NPOSTFIX) + lcode + self.NDIRECT + 1 + return (0,distance) + + def mnemonic(self, index, verbose=False): + """Give mnemonic representation of meaning. + verbose compresses strings of x's + """ + if index<16: + return ['last', '2last', '3last', '4last', + 'last-1', 'last+1', 'last-2', 'last+2', 'last-3', 'last+3', + '2last-1', '2last+1', '2last-2', '2last+2', '2last-3', '2last+3' + ][index] + if index<16+self.NDIRECT: + return str(index-16) + #construct strings like "1xx01-15" + index -= self.NDIRECT+16 + hcode = index >> self.NPOSTFIX + lcode = index & (1<<self.NPOSTFIX)-1 + if self.NPOSTFIX: formatString = '1{0}{1}{2:0{3}b}{4:+d}' + else: formatString = '1{0}{1}{4:+d}' + return formatString.format( + hcode&1, + 'x'*(2+hcode>>1) if hcode<13 or verbose else '[{}*x]'.format(2+hcode>>1), + lcode, self.NPOSTFIX, + self.NDIRECT+1-(4<<self.NPOSTFIX)) + + def explanation(self, index, extra): + """ + >>> d = DistanceAlphabet('D', NPOSTFIX=2, NDIRECT=10) + >>> d[55].explanation(13) + '11[1101]01-5: [0]+240' + """ + extraBits = self.extraBits(index) + extraString = '[{:0{}b}]'.format(extra, extraBits) + return '{0}: [{1[0]}]{1[1]:+d}'.format( + self.mnemonic(index, True).replace('x'*(extraBits or 1), extraString), + self.value(index, extra)) + +#Classes for doing actual work------------------------------------------ +class ContextModeKeeper: + """For computing the literal context mode. + You feed it characters, and it computes indices in the context map. + """ + def __init__(self, mode): + self.chars = deque([0,0], maxlen=2) + self.mode = mode + + def setContextMode(self, mode): + """Switch to given context mode (0..3)""" + self.mode = mode + def getIndex(self): + if self.mode==0: #LSB6 + return self.chars[1]&0x3f + elif self.mode==1: #MSB6 + return self.chars[1]>>2 + elif self.mode==2: #UTF8: character class of previous and a bit of the second + p2,p1 = self.chars + return self.lut0[p1]|self.lut1[p2] + elif self.mode==3: #Signed: initial bits of last two bytes + p2,p1 = self.chars + return self.lut2[p1]<<3|self.lut2[p2] + + def add(self, index): + """Adjust the context for output char (as int).""" + self.chars.append(index) + + #0: control #16: quote #32: ,:; #48: AEIOU + #4: tab/lf/cr #20: % #36: . #52: BC..Z + #8: space #24: (<[{ #40: = #56: aeiou + #12:!#$&*+-/?@| #28: )>]} #44: 0-9 #60: bc..z + lut0 = [0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0, 0, 4, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 8, 12, 16, 12, 12, 20, 12, 16, 24, 28, 12, 12, 32, 12, 36, 12, + 44, 44, 44, 44, 44, 44, 44, 44, 44, 44, 32, 32, 24, 40, 28, 12, + 12, 48, 52, 52, 52, 48, 52, 52, 52, 48, 52, 52, 52, 52, 52, 48, + 52, 52, 52, 52, 52, 48, 52, 52, 52, 52, 52, 24, 12, 28, 12, 12, + 12, 56, 60, 60, 60, 56, 60, 60, 60, 56, 60, 60, 60, 60, 60, 56, + 60, 60, 60, 60, 60, 56, 60, 60, 60, 60, 60, 24, 12, 28, 12, 0 + ]+[0,1]*32+[2,3]*32 + #0: space 1:punctuation 2:digit/upper 3:lower + lut1 = [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, 1, + 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, + 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 1, + 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, 1, 1, 1, 0 + ]+[0]*96+[2]*32 + #initial bits: 8*0, 4*0, 2*0, 1*0, 1*1, 2*1, 4*1, 8*1 + lut2 = [0]+[1]*15+[2]*48+[3]*64+[4]*64+[5]*48+[6]*15+[7] + assert len(lut0)==len(lut1)==len(lut2)==256 + +class WordList: + """Word list. + >>> WordList().word(7, 35555) + b'Program to ' + """ + NDBITS = [0, 0, 0, 0, 10, 10, 11, 11, 10, 10, + 10, 10, 10, 9, 9, 8, 7, 7, 8, 7, + 7, 6, 6, 5, 5] + def __init__(self): + self.file = open('dict', 'rb') + self.compileActions() + + def word(self, size, dist): + """Get word + """ + #split dist in index and action + ndbits = self.NDBITS[size] + index = dist&(1<<ndbits)-1 + action = dist>>ndbits + #compute position in file + position = sum(n<<self.NDBITS[n] for n in range(4,size))+size*index + self.file.seek(position) + return self.doAction(self.file.read(size), action) + + def upperCase1(self, word): + word = word.decode('utf8') + word = word[0].upper()+word[1:] + return word.encode('utf8') + + + #Super compact form of action table. + #_ means space, .U means UpperCaseAll, U(w) means UpperCaseFirst + actionTable = r""" + 0:w 25:w+_for_ 50:w+\n\t 75:w+. This_100:w+ize_ + 1:w+_ 26:w[3:] 51:w+: 76:w+, 101:w.U+. + 2:_+w+_ 27:w[:-2] 52:_+w+._ 77:.+w+_ 102:\xc2\xa0+w + 3:w[1:] 28:w+_a_ 53:w+ed_ 78:U(w)+( 103:_+w+, + 4:U(w)+_ 29:w+_that_ 54:w[9:] 79:U(w)+. 104:U(w)+=" + 5:w+_the_ 30:_+U(w) 55:w[7:] 80:w+_not_ 105:w.U+=" + 6:_+w 31:w+._ 56:w[:-6] 81:_+w+=" 106:w+ous_ + 7:s_+w+_ 32:.+w 57:w+( 82:w+er_ 107:w.U+,_ + 8:w+_of_ 33:_+w+,_ 58:U(w)+,_ 83:_+w.U+_ 108:U(w)+=\' + 9:U(w) 34:w[4:] 59:w[:-8] 84:w+al_ 109:_+U(w)+, + 10:w+_and_ 35:w+_with_ 60:w+_at_ 85:_+w.U 110:_+w.U+=" + 11:w[2:] 36:w+\' 61:w+ly_ 86:w+=\' 111:_+w.U+,_ + 12:w[:-1] 37:w+_from_ 62:_the_+w+_of_ 87:w.U+" 112:_+w.U+, + 13:,_+w+_ 38:w+_by_ 63:w[:-5] 88:U(w)+._ 113:w.U+( + 14:w+,_ 39:w[5:] 64:w[:-9] 89:_+w+( 114:w.U+._ + 15:_+U(w)+_ 40:w[6:] 65:_+U(w)+,_ 90:w+ful_ 115:_+w.U+. + 16:w+_in_ 41:_the_+w 66:U(w)+" 91:_+U(w)+._116:w.U+=\' + 17:w+_to_ 42:w[:-4] 67:.+w+( 92:w+ive_ 117:_+w.U+._ + 18:e_+w+_ 43:w+. The_ 68:w.U+_ 93:w+less_ 118:_+U(w)+=" + 19:w+" 44:w.U 69:U(w)+"> 94:w.U+\' 119:_+w.U+=\' + 20:w+. 45:w+_on_ 70:w+=" 95:w+est_ 120:_+U(w)+=\' + 21:w+"> 46:w+_as_ 71:_+w+. 96:_+U(w)+. + 22:w+\n 47:w+_is_ 72:.com/+w 97:w.U+"> + 23:w[:-3] 48:w[:-7] 98:_+w+=\' + 24:w+] 49:w[:-1]+ing_ 74:U(w)+\' 99:U(w)+, + """ + + def compileActions(self): + """Build the action table from the text above + """ + import re + self.actionList = actions = [None]*121 + #Action 73, which is too long, looks like this when expanded: + actions[73] = "b' the '+w+b' of the '" + #find out what the columns are + actionLines = self.actionTable.splitlines() + colonPositions = [m.start() + for m in re.finditer(':',actionLines[1]) + ]+[100] + columns = [(colonPositions[i]-3,colonPositions[i+1]-3) + for i in range(len(colonPositions)-1)] + for line in self.actionTable.splitlines(keepends=False): + for start,end in columns: + action = line[start:end] + #skip empty actions + if not action or action.isspace(): continue + #chop it up, and check if the colon is properly placed + index, colon, action = action[:3], action[3], action[4:] + assert colon==':' + #remove filler spaces at right + action = action.rstrip() + #replace space symbols + action = action.replace('_', ' ') + wPos = action.index('w') + #add quotes around left string when present + #translation: any pattern from beginning, up to + #(but not including) a + following by a w later on + action = re.sub(r"^(.*)(?=\+[U(]*w)", r"b'\1'", action) + #add quotes around right string when present + #translation: anything with a w in it, followed by a + + #and a pattern up to the end + #(there is no variable lookbehind assertion, + #so we have to copy the pattern) + action = re.sub(r"(w[[:\-1\]).U]*)\+(.*)$", r"\1+b'\2'", action) + #expand shortcut for uppercaseAll + action = action.replace(".U", ".upper()") + #store action + actions[int(index)] = action + + def doAction(self, w, action): + """Perform the proper action + """ + #set environment for the UpperCaseFirst + U = self.upperCase1 + return eval(self.actionList[action], locals()) + +class Layout: + """Class to layout the output. + """ + #display width of hexdata+bitdata + width = 25 + #general + def __init__(self, stream): + self.stream = stream + self.bitPtr = self.width + + def makeHexData(self, pos): + """Produce hex dump of all data containing the bits + from pos to stream.pos + """ + firstAddress = pos+7>>3 + lastAddress = self.stream.pos+7>>3 + return ''.join(map('{:02x} '.format, + self.stream.data[firstAddress:lastAddress])) + + def formatBitData(self, pos, width1, width2=0): + """Show formatted bit data: + Bytes are separated by commas + whole bytes are displayed in hex + >>> Layout(olleke).formatBitData(6, 2, 16) + '|00h|2Eh,|00' + >>> Layout(olleke).formatBitData(4, 1, 0) + '1' + """ + result = [] + #make empty prefix code explicit + if width1==0: result = ['()', ','] + for width in width1, width2: + #skip empty width2 + if width==0: continue + #build result backwards in a list + while width>0: + availableBits = 8-(pos&7) + if width<availableBits: + #read partial byte, beginning nor ending at boundary + data = self.stream.data[pos>>3] >> (pos&7) & (1<<width)-1 + result.append('{:0{}b}'.format(data, width)) + elif availableBits<8: + #read rest of byte, ending at boundary + data = self.stream.data[pos>>3] >> (pos&7) + result.append('|{:0{}b}'.format(data, availableBits)) + else: + #read whole byte (in hex), beginning and ending at boundary + data = self.stream.data[pos>>3] + result.append('|{:02X}h'.format(data)) + width -= availableBits + pos += availableBits + #if width overshot from the availableBits subtraction, fix it + pos += width + #add comma to separate fields + result.append(',') + #concatenate pieces, reversed, skipping the last space + return ''.join(result[-2::-1]) + + def readPrefixCode(self, alphabet): + """give alphabet the prefix code that is read from the stream + Called for the following alphabets, in this order: + The alphabet in question must have a "logical" order, + otherwise the assignment of symbols doesn't work. + """ + mode, numberOfSymbols = self.verboseRead(PrefixCodeHeader(alphabet.name)) + if mode=='Complex': + #for a complex code, numberOfSymbols means hskip + self.readComplexCode(numberOfSymbols, alphabet) + return alphabet + else: + table = [] + #Set table of lengths for mnemonic function + lengths = [[0], [1,1], [1,2,2], '????'][numberOfSymbols-1] + #adjust mnemonic function of alphabet class + def myMnemonic(index): + return '{} bit{}: {}'.format( + lengths[i], + '' if lengths[i]==1 else 's', + alphabet.__class__.mnemonic(alphabet, index) + ) + alphabet.mnemonic = myMnemonic + for i in range(numberOfSymbols): + table.append(self.verboseRead(alphabet, skipExtra=True).index) + #restore mnemonic + del alphabet.mnemonic + if numberOfSymbols==4: + #read tree shape to redefine lengths + lengths = self.verboseRead(TreeShapeAlhabet()) + #construct the alphabet prefix code + alphabet.setLength(dict(zip(table, lengths))) + return alphabet + + def readComplexCode(self, hskip, alphabet): + """Read complex code""" + stream = self.stream + #read the lengths for the length code + lengths = [1,2,3,4,0,5,17,6,16,7,8,9,10,11,12,13,14,15][hskip:] + codeLengths = {} + total = 0 + lol = LengthOfLengthAlphabet('##'+alphabet.name) + #lengthCode will be used for coding the lengths of the new code + #we use it for display until now; definition comes below + lengthCode = LengthAlphabet('#'+alphabet.name) + lengthIter = iter(lengths) + lengthsLeft = len(lengths) + while total<32 and lengthsLeft>0: + lengthsLeft -= 1 + newSymbol = next(lengthIter) + lol.description = str(lengthCode[newSymbol]) + length = self.verboseRead(lol) + if length: + codeLengths[newSymbol] = length + total += 32>>length + if total>32: raise ValueError("Stream format") + if len(codeLengths)==1: codeLengths[list(codeLengths.keys())[0]] = 0 + #Now set the encoding of the lengthCode + lengthCode.setLength(codeLengths) + print("***** Lengths for {} will be coded as:".format(alphabet.name)) + lengthCode.showCode() + #Now determine the symbol lengths with the lengthCode + symbolLengths = {} + total = 0 + lastLength = 8 + alphabetIter = iter(alphabet) + while total<32768: + #look ahead to see what is going to happen + length = lengthCode.decodePeek( + self.stream.peek(lengthCode.maxLength))[1].index + #in every branch, set lengthCode.description to explanatory text + #lengthCode calls format(symbol, extra) with this string + if length==0: + symbol = next(alphabetIter) + lengthCode.description = 'symbol {} unused'.format(symbol) + self.verboseRead(lengthCode) + #unused symbol + continue + if length==16: + lengthCode.description = \ + '{1}+3 symbols of length '+str(lastLength) + extra = self.verboseRead(lengthCode) + #scan series of 16s (repeat counts) + #start with repeat count 2 + repeat = 2 + startSymbol = next(alphabetIter) + endSymbol = next(alphabetIter) + symbolLengths[startSymbol.index] = \ + symbolLengths[endSymbol.index] = lastLength + #count the two just defined symbols + total += 2*32768>>lastLength + #note: loop may end because we're there + #even if a 16 _appears_ to follow + while True: + #determine last symbol + oldRepeat = repeat + repeat = (repeat-2<<2)+extra+3 + #read as many symbols as repeat increased + for i in range(oldRepeat, repeat): + endSymbol = next(alphabetIter) + symbolLengths[endSymbol.index] = lastLength + #compute new total; it may be end of loop + total += (repeat-oldRepeat)*32768>>lastLength + if total>=32768: break + #see if there is more to do + length = lengthCode.decodePeek( + self.stream.peek(lengthCode.maxLength))[1].index + if length!=16: break + lengthCode.description = 'total {}+{{1}} symbols'.format( + (repeat-2<<2)+3) + extra = self.verboseRead(lengthCode) + elif length==17: + #read, and show explanation + lengthCode.description = '{1}+3 unused' + extra = self.verboseRead(lengthCode) + #scan series of 17s (groups of zero counts) + #start with repeat count 2 + repeat = 2 + startSymbol = next(alphabetIter) + endSymbol = next(alphabetIter) + #note: loop will not end with total==32768, + #since total doesn't change here + while True: + #determine last symbol + oldRepeat = repeat + repeat = (repeat-2<<3)+extra+3 + #read as many symbols as repeat increases + for i in range(repeat-oldRepeat): + endSymbol = next(alphabetIter) + #see if there is more to do + length = lengthCode.decodePeek( + self.stream.peek(lengthCode.maxLength))[1].index + if length!=17: break + lengthCode.description = 'total {}+{{1}} unused'.format( + (repeat-2<<3)+3) + extra = self.verboseRead(lengthCode) + else: + symbol = next(alphabetIter) + #double braces for format + char = str(symbol) + if char in '{}': char *= 2 + lengthCode.description = \ + 'Length for {} is {{0.index}} bits'.format(char) + #output is not needed (will be 0) + self.verboseRead(lengthCode) + symbolLengths[symbol.index] = length + total += 32768>>length + lastLength = length + assert total==32768 + alphabet.setLength(symbolLengths) + print('End of table. Prefix code '+alphabet.name+':') + alphabet.showCode() + + #stream + def processStream(self): + """Process a brotli stream. + """ + print('addr hex{:{}s}binary context explanation'.format( + '', self.width-10)) + print('Stream header'.center(60, '-')) + self.windowSize = self.verboseRead(WindowSizeAlphabet()) + print('Metablock header'.center(60, '=')) + self.ISLAST = False + self.output = bytearray() + while not self.ISLAST: + self.ISLAST = self.verboseRead( + BoolCode('LAST', description="Last block")) + if self.ISLAST: + if self.verboseRead( + BoolCode('EMPTY', description="Empty block")): break + if self.metablockLength(): continue + if not self.ISLAST and self.uncompressed(): continue + print('Block type descriptors'.center(60, '-')) + self.numberOfBlockTypes = {} + self.currentBlockCounts = {} + self.blockTypeCodes = {} + self.blockCountCodes = {} + for blockType in (L,I,D): self.blockType(blockType) + print('Distance code parameters'.center(60, '-')) + self.NPOSTFIX, self.NDIRECT = self.verboseRead(DistanceParamAlphabet()) + self.readLiteralContextModes() + print('Context maps'.center(60, '-')) + self.cmaps = {} + #keep the number of each kind of prefix tree for the last loop + numberOfTrees = {I: self.numberOfBlockTypes[I]} + for blockType in (L,D): + numberOfTrees[blockType] = self.contextMap(blockType) + print('Prefix code lists'.center(60, '-')) + self.prefixCodes = {} + for blockType in (L,I,D): + self.readPrefixArray(blockType, numberOfTrees[blockType]) + self.metablock() + + #metablock header + def verboseRead(self, alphabet, context='', skipExtra=False): + """Read symbol and extra from stream and explain what happens. + Returns the value of the symbol + >>> olleke.pos = 0 + >>> l = Layout(olleke) + >>> l.verboseRead(WindowSizeAlphabet()) + 0000 1b 1011 WSIZE windowsize=(1<<22)-16=4194288 + 4194288 + """ + #TODO 2: verbosity level, e.g. show only codes and maps in header + stream = self.stream + pos = stream.pos + if skipExtra: + length, symbol = alphabet.readTuple(stream) + extraBits, extra = 0, None + else: + length, symbol, extraBits, extra = alphabet.readTupleAndExtra( + stream) + #fields: address, hex data, binary data, name of alphabet, explanation + hexdata = self.makeHexData(pos) + addressField = '{:04x}'.format(pos+7>>3) if hexdata else '' + bitdata = self.formatBitData(pos, length, extraBits) + #bitPtr moves bitdata so that the bytes are easier to read + #jump back to right if a new byte starts + if '|' in bitdata[1:]: + #start over on the right side + self.bitPtr = self.width + fillWidth = self.bitPtr-(len(hexdata)+len(bitdata)) + if fillWidth<0: fillWidth = 0 + print('{:<5s} {:<{}s} {:7s} {}'.format( + addressField, + hexdata+' '*fillWidth+bitdata, self.width, + context+alphabet.name, + symbol if skipExtra else symbol.explanation(extra), + )) + #jump to the right if we started with a '|' + #because we didn't jump before printing + if bitdata.startswith('|'): self.bitPtr = self.width + else: self.bitPtr -= len(bitdata) + return symbol if skipExtra else symbol.value(extra) + + def metablockLength(self): + """Read MNIBBLES and meta block length; + if empty block, skip block and return true. + """ + self.MLEN = self.verboseRead(MetablockLengthAlphabet()) + if self.MLEN: + return False + #empty block; skip and return False + self.verboseRead(ReservedAlphabet()) + MSKIP = self.verboseRead(SkipLengthAlphabet()) + self.verboseRead(FillerAlphabet(streamPos=self.stream.pos)) + self.stream.pos += 8*MSKIP + print("Skipping to {:x}".format(self.stream.pos>>3)) + return True + + def uncompressed(self): + """If true, handle uncompressed data + """ + ISUNCOMPRESSED = self.verboseRead( + BoolCode('UNCMPR', description='Is uncompressed?')) + if ISUNCOMPRESSED: + self.verboseRead(FillerAlphabet(streamPos=self.stream.pos)) + print('Uncompressed data:') + self.output += self.stream.readBytes(self.MLEN) + print(outputFormatter(self.output[-self.MLEN:])) + return ISUNCOMPRESSED + + def blockType(self, kind): + """Read block type switch descriptor for given kind of blockType.""" + NBLTYPES = self.verboseRead(TypeCountAlphabet( + 'BT#'+kind[0].upper(), + description='{} block types'.format(kind), + )) + self.numberOfBlockTypes[kind] = NBLTYPES + if NBLTYPES>=2: + self.blockTypeCodes[kind] = self.readPrefixCode( + BlockTypeAlphabet('BT'+kind[0].upper(), NBLTYPES)) + self.blockCountCodes[kind] = self.readPrefixCode( + BlockCountAlphabet('BC'+kind[0].upper())) + blockCount = self.verboseRead(self.blockCountCodes[kind]) + else: + blockCount = 1<<24 + self.currentBlockCounts[kind] = blockCount + + def readLiteralContextModes(self): + """Read literal context modes. + LSB6: lower 6 bits of last char + MSB6: upper 6 bits of last char + UTF8: rougly dependent on categories: + upper 4 bits depend on category of last char: + control/whitespace/space/ punctuation/quote/%/open/close/ + comma/period/=/digits/ VOWEL/CONSONANT/vowel/consonant + lower 2 bits depend on category of 2nd last char: + space/punctuation/digit or upper/lowercase + signed: hamming weight of last 2 chars + """ + print('Context modes'.center(60, '-')) + self.literalContextModes = [] + for i in range(self.numberOfBlockTypes[L]): + self.literalContextModes.append( + self.verboseRead(LiteralContextMode(number=i))) + + def contextMap(self, kind): + """Read context maps + Returns the number of differnt values on the context map + (In other words, the number of prefix trees) + """ + NTREES = self.verboseRead(TypeCountAlphabet( + kind[0].upper()+'T#', + description='{} prefix trees'.format(kind))) + mapSize = {L:64, D:4}[kind] + if NTREES<2: + self.cmaps[kind] = [0]*mapSize + else: + #read CMAPkind + RLEMAX = self.verboseRead(RLEmaxAlphabet( + 'RLE#'+kind[0].upper(), + description=kind+' context map')) + alphabet = TreeAlphabet('CM'+kind[0].upper(), NTREES=NTREES, RLEMAX=RLEMAX) + cmapCode = self.readPrefixCode(alphabet) + tableSize = mapSize*self.numberOfBlockTypes[kind] + cmap = [] + while len(cmap)<tableSize: + cmapCode.description = 'map {}, entry {}'.format( + *divmod(len(cmap), mapSize)) + count, value = self.verboseRead(cmapCode) + cmap.extend([value]*count) + assert len(cmap)==tableSize + IMTF = self.verboseRead(BoolCode('IMTF', description='Apply inverse MTF')) + if IMTF: + self.IMTF(cmap) + if kind==L: + print('Context maps for literal data:') + for i in range(0, len(cmap), 64): + print(*( + ''.join(map(str, cmap[j:j+8])) + for j in range(i, i+64, 8) + )) + else: + print('Context map for distances:') + print(*( + ''.join(map('{:x}'.format, cmap[i:i+4])) + for i in range(0, len(cmap), 4) + )) + self.cmaps[kind] = cmap + return NTREES + + @staticmethod + def IMTF(v): + """In place inverse move to front transform. + """ + #mtf is initialized virtually with range(infinity) + mtf = [] + for i, vi in enumerate(v): + #get old value from mtf. If never seen, take virtual value + try: value = mtf.pop(vi) + except IndexError: value = vi + #put value at front + mtf.insert(0, value) + #replace transformed value + v[i] = value + + def readPrefixArray(self, kind, numberOfTrees): + """Read prefix code array""" + prefixes = [] + for i in range(numberOfTrees): + if kind==L: alphabet = LiteralAlphabet(i) + elif kind==I: alphabet = InsertAndCopyAlphabet(i) + elif kind==D: alphabet = DistanceAlphabet( + i, NPOSTFIX=self.NPOSTFIX, NDIRECT=self.NDIRECT) + self.readPrefixCode(alphabet) + prefixes.append(alphabet) + self.prefixCodes[kind] = prefixes + + #metablock data + def metablock(self): + """Process the data. + Relevant variables of self: + numberOfBlockTypes[kind]: number of block types + currentBlockTypes[kind]: current block types (=0) + literalContextModes: the context modes for the literal block types + currentBlockCounts[kind]: counters for block types + blockTypeCodes[kind]: code for block type + blockCountCodes[kind]: code for block count + cmaps[kind]: the context maps (not for I) + prefixCodes[kind][#]: the prefix codes + lastDistances: the last four distances + lastChars: the last two chars + output: the result + """ + print('Meta block contents'.center(60, '=')) + self.currentBlockTypes = {L:0, I:0, D:0, pL:1, pI:1, pD:1} + self.lastDistances = deque([17,16,11,4], maxlen=4) + #the current context mode is for block type 0 + self.contextMode = ContextModeKeeper(self.literalContextModes[0]) + wordList = WordList() + + #setup distance callback function + def distanceCallback(symbol, extra): + "callback function for displaying decoded distance" + index, offset = symbol.value(extra) + if index: + #recent distance + distance = self.lastDistances[-index]+offset + return 'Distance: {}last{:+d}={}'.format(index, offset, distance) + #absolute value + if offset<=maxDistance: + return 'Absolute value: {} (pos {})'.format(offset, maxDistance-offset) + #word list value + action, word = divmod(offset-maxDistance, 1<<wordList.NDBITS[copyLen]) + return '{}-{} gives word {},{} action {}'.format( + offset, maxDistance, copyLen, word, action) + for dpc in self.prefixCodes[D]: dpc.callback = distanceCallback + + blockLen = 0 + #there we go + while blockLen<self.MLEN: + #get insert© command + litLen, copyLen, dist0Flag = self.verboseRead( + self.prefixCodes[I][ + self.figureBlockType(I)]) + #literal data + for i in range(litLen): + bt = self.figureBlockType(L) + cm = self.contextMode.getIndex() + ct = self.cmaps[L][bt<<6|cm] + char = self.verboseRead( + self.prefixCodes[L][ct], + context='{},{}='.format(bt,cm)) + self.contextMode.add(char) + self.output.append(char) + blockLen += litLen + #check if we're done + if blockLen>=self.MLEN: return + #distance + #distances are computed relative to output length, at most window size + maxDistance = min(len(self.output), self.windowSize) + if dist0Flag: + distance = self.lastDistances[-1] + else: + bt = self.figureBlockType(D) + cm = {2:0, 3:1, 4:2}.get(copyLen, 3) + ct = self.cmaps[D][bt<<2|cm] + index, offset = self.verboseRead( + self.prefixCodes[D][ct], + context='{},{}='.format(bt,cm)) + distance = self.lastDistances[-index]+offset if index else offset + if index==1 and offset==0: + #to make sure distance is not put in last distance list + dist0Flag = True + if distance<=maxDistance: + #copy from output + for i in range( + maxDistance-distance, + maxDistance-distance+copyLen): + self.output.append(self.output[i]) + if not dist0Flag: self.lastDistances.append(distance) + comment = 'Seen before' + else: + #fetch from wordlist + newWord = wordList.word(copyLen, distance-maxDistance-1) + self.output.extend(newWord) + #adjust copyLen to reflect actual new data + copyLen = len(newWord) + comment = 'From wordlist' + blockLen += copyLen + print(' '*40, + comment, + ': "', + outputFormatter(self.output[-copyLen:]), + '"', + sep='') + self.contextMode.add(self.output[-2]) + self.contextMode.add(self.output[-1]) + + def figureBlockType(self, kind): + counts, types = self.currentBlockCounts, self.currentBlockTypes + if counts[kind]==0: + newType = self.verboseRead(self.blockTypeCodes[kind]) + if newType==-2: newType = types['P'+kind] + elif newType==-1: + newType = (types[kind]+1)%self.numberOfBlockTypes[kind] + types['P'+kind] = types[kind] + types[kind] = newType + counts[kind] = self.verboseRead(self.blockCountCodes[kind]) + counts[kind] -=1 + return types[kind] + +__test__ = { +'BitStream': """ + >>> bs = BitStream(b'Jurjen') + >>> bs.readBytes(2) + b'Ju' + >>> bs.read(6) #r=01110010 + 50 + >>> bs + BitStream(pos=2:6) + >>> bs.peek(5) #j=01101010 + 9 + >>> bs.readBytes(2) + Traceback (most recent call last): + ... + ValueError: readBytes: need byte boundary + """, + +'Symbol': """ + >>> a=Symbol(MetablockLengthAlphabet(),5) + >>> len(a) + 2 + >>> int(a) + 5 + >>> a.bitPattern() + '01' + >>> a.value(200000) + 200001 + >>> a.explanation(300000) + 'data length: 493e0h+1=300001' + """, + +'RangeDecoder': """ + >>> a=RangeDecoder(bitLength=3) + >>> len(a) + 8 + >>> a.name='t' + >>> list(a) + [Symbol(t, 0), Symbol(t, 1), Symbol(t, 2), Symbol(t, 3), Symbol(t, 4), Symbol(t, 5), Symbol(t, 6), Symbol(t, 7)] + >>> a[2] + Symbol(t, 2) + >>> a.bitPattern(4) + '100' + >>> a.length(2) + 3 + >>> a.decodePeek(15) + (3, Symbol(t, 7)) + >>> + + """, + +'PrefixDecoder': """ + >>> a=PrefixDecoder(decodeTable={0:1,1:2,3:3,7:4}) + >>> len(a) + 4 + >>> a.name='t' + >>> list(a) + [Symbol(t, 1), Symbol(t, 2), Symbol(t, 3), Symbol(t, 4)] + >>> a.decodePeek(22) + (1, Symbol(t, 1)) + >>> a.decodePeek(27) + (3, Symbol(t, 3)) + >>> a.length(1) + 1 + >>> a.length(4) + 3 + """, + +'Code': """ + >>> a=Code('t',alphabetSize=10) + >>> len(a) + 10 + >>> a.showCode() + 0000:0 0001:1 0010:2 0011:3 0100:4 0101:5 0110:6 0111:7 1000:8 1001:9 + >>> a.setLength({2:1,3:2,5:3,6:3}) + >>> a.showCode() + 0:2 01:3 011:5 111:6 + >>> len(a) + 4 + >>> def callback(i): return 'call{}back'.format(i) + >>> a=Code('t',callback=callback,bitLength=3) + >>> a[6].explanation() + 'call6back' + """, + +'WithExtra': """ + >>> class A(WithExtra): + ... extraTable = [0,1,1,2,2] + >>> a=A('t',alphabetSize=5) + >>> a[1] + Symbol(t, 1) + >>> a.extraBits(2) + 1 + >>> a.mnemonic(4) + '4' + >>> a.readTupleAndExtra(BitStream(b'\x5b')) + (3, Symbol(t, 3), 2, 3) + """, + +'BoolCode': """ + >>> BoolCode('test')[0].explanation() + '0: False' + """, + +'Enumerator': """ + >>> class A(Enumerator): + ... extraTable = [0,1,1,2,2] + ... value0=3 + >>> a=A(alphabetLength=5) + >>> a.value(3) + Traceback (most recent call last): + ... + TypeError: value() missing 1 required positional argument: 'extra' + >>> a.explanation(3,4) + 'xx 011: 8-11; 8+4=12' + """, + +'WindowSizeAlphabet': """ + >>> windowSizeAlphabet = WindowSizeAlphabet() + >>> windowSizeAlphabet[0] + Traceback (most recent call last): + ... + ValueError: No symbol WindowSizeAlphabet[0] + >>> len(windowSizeAlphabet) + 16 + >>> windowSizeAlphabet[21] + Symbol(WSIZE, 21) + >>> windowSizeAlphabet[21].bitPattern() + '1001' + >>> windowSizeAlphabet[21].extraBits() + 0 + >>> windowSizeAlphabet[21].index + 21 + >>> windowSizeAlphabet[10].value() + 1008 + >>> windowSizeAlphabet[10].explanation() + 'windowsize=(1<<10)-16=1008' + >>> windowSizeAlphabet.showCode() + 0:65520 1100001:16368 1110001:32752 0011:262128 + 0000001:131056 0010001:None 1001:2097136 1011:4194288 + 1000001:4080 1010001:8176 0101:524272 0111:1048560 + 0100001:1008 0110001:2032 1101:8388592 1111:16777200 + """, + +'TypeCountAlphabet': """ + >>> typeCountAlphabet = TypeCountAlphabet(description='bananas') + >>> len(typeCountAlphabet) + 9 + >>> typeCountAlphabet[3] + Symbol(BT#, 3) + >>> typeCountAlphabet[9] + Traceback (most recent call last): + ... + ValueError: No symbol TypeCountAlphabet[9] + >>> print(typeCountAlphabet[3]) + xx,0101 + >>> typeCountAlphabet[8].value(127) + 256 + >>> typeCountAlphabet[4].explanation(2) + 'xxx,0111: 11 bananas' + >>> typeCountAlphabet[0].explanation() + '0: 1 banana' + """, + +'DistanceParamAlphabet': """ + >>> dpa = DistanceParamAlphabet() + >>> dpa.showCode() + 00:PF0 01:PF1 10:PF2 11:PF3 + >>> dpa.readTupleAndExtra(BitStream(b'\\x29')) + (2, Symbol(DIST, 1), 4, 10) + >>> dpa.explanation(2, 5) + '2 postfix bits and 0101<<2=20 direct codes' + """, + +'LiteralAlphabet': """ + >>> LiteralAlphabet(-1).showCode() #doctest: +ELLIPSIS + 00000000:\\x00 00110100:4 01101000:h 10011100:\\x9c 11010000:\\xd0 + 00000001:\\x01 00110101:5 01101001:i 10011101:\\x9d 11010001:\\xd1 + 00000010:\\x02 00110110:6 01101010:j 10011110:\\x9e 11010010:\\xd2 + ... + 00101111:/ 01100011:c 10010111:\\x97 11001011:\\xcb 11111111:\\xff + 00110000:0 01100100:d 10011000:\\x98 11001100:\\xcc + 00110001:1 01100101:e 10011001:\\x99 11001101:\\xcd + 00110010:2 01100110:f 10011010:\\x9a 11001110:\\xce + 00110011:3 01100111:g 10011011:\\x9b 11001111:\\xcf + """, + +'BlockCountAlphabet': """ + >>> bc=BlockCountAlphabet('BCL') + >>> len(bc) + 26 + >>> bs=BitStream(b'\\x40\\x83\\xc8\\x59\\12\\x02') + >>> x = bc.readTupleAndExtra(bs); x[1].explanation(x[3]) + 'Block count: xx 00000: 1-4; 1+2=3' + >>> x = bc.readTupleAndExtra(bs); x[1].explanation(x[3]) + 'Block count: xxx 00110: 33-40; 33+0=33' + >>> x = bc.readTupleAndExtra(bs); x[1].explanation(x[3]) + 'Block count: xxxxxx 10001: 305-368; 305+28=333' + >>> x = bc.readTupleAndExtra(bs); x[1].explanation(x[3]) + 'Block count: xxxxxxxxxxx 10110: 2289-4336; 2289+1044=3333' + """, + +'Layout': """ + >>> olleke.pos = 0 + >>> l = Layout(olleke) + >>> l.verboseRead(WindowSizeAlphabet()) + 0000 1b 1011 WSIZE windowsize=(1<<22)-16=4194288 + 4194288 + >>> l.verboseRead(BoolCode('LAST', description="Last block")) + 1 LAST Last block: 1: True + True + >>> l.verboseRead(BoolCode('EMPTY', description="Empty block")) + 0 EMPTY Empty block: 0: False + False + >>> l.verboseRead(MetablockLengthAlphabet()) + 0001 2e 00 |00h|2Eh,|00 MLEN data length: 002eh+1=47 + 47 + >>> olleke.pos = 76 + >>> l = Layout(olleke) + >>> x = l.verboseRead(DistanceAlphabet(0,NPOSTFIX=0,NDIRECT=0), skipExtra=True) + 000a 82 10|1100 D0 10[15*x]-3 + >>> x.explanation(0x86a3) + '10[1000011010100011]-3: [0]+100000' + """, + +'olleke': """ + >>> olleke.pos = 0 + >>> try: Layout(olleke).processStream() + ... except NotImplementedError: pass + ... #doctest: +REPORT_NDIFF + addr hex binary context explanation + -----------------------Stream header------------------------ + 0000 1b 1011 WSIZE windowsize=(1<<22)-16=4194288 + ======================Metablock header====================== + 1 LAST Last block: 1: True + 0 EMPTY Empty block: 0: False + 0001 2e 00 |00h|2Eh,|00 MLEN data length: 002eh+1=47 + -------------------Block type descriptors------------------- + 0003 00 0 BT#L 0: 1 literal block type + 0 BT#I 0: 1 insert© block type + 0 BT#D 0: 1 distance block type + ------------------Distance code parameters------------------ + 0004 44 0|000,00 DIST 0 postfix bits and 0000<<0=0 direct codes + -----------------------Context modes------------------------ + 10 LC0 Context mode for type 0: 2(UTF8) + ------------------------Context maps------------------------ + 0 LT# 0: 1 literal prefix tree + 0 DT# 0: 1 distance prefix tree + ---------------------Prefix code lists---------------------- + 10 PFX L0 is complex with lengths 3,4,0,5,17... + 0005 4f 1|0 ##L0 len 3: coded with 3 bits + 0111 ##L0 len 4: coded with 1 bits + 10 ##L0 unused: coded with 3 bits + 0006 d6 0|0 ##L0 len 5: skipped + 011 ##L0 zero xxx: coded with 2 bits + ***** Lengths for L0 will be coded as: + 0:len 4 01:zero xxx 011:unused 111:len 3 + 0007 95 1|11,01 #L0 7+3 unused + 0 #L0 Length for \\n is 4 bits + 001,01 #L0 1+3 unused + 0008 44 010,0|1 #L0 total 19+2 unused + 0 #L0 Length for " " is 4 bits + 0 #L0 Length for ! is 4 bits + 0009 cb 011,|01 #L0 3+3 unused + |110,01 #L0 total 35+6 unused + 000a 82 0 #L0 Length for K is 4 bits + 000,01 #L0 0+3 unused + 0 #L0 Length for O is 4 bits + 000b 4d 01|1 #L0 symbol P unused + 011 #L0 symbol Q unused + 0 #L0 Length for R is 4 bits + 000c 88 000,|01 #L0 0+3 unused + |100,01 #L0 total 11+4 unused + 000d b6 0 #L0 Length for b is 4 bits + 011 #L0 symbol c unused + 011 #L0 symbol d unused + 000e 27 11|1 #L0 Length for e is 3 bits + 010,01 #L0 2+3 unused + |0 #L0 Length for k is 4 bits + 000f 1f 111 #L0 Length for l is 3 bits + 011 #L0 symbol m unused + 0 #L0 Length for n is 4 bits + |0 #L0 Length for o is 4 bits + 0010 c1 000,01 #L0 0+3 unused + 0 #L0 Length for s is 4 bits + 0011 b4 0|11 #L0 symbol t unused + 0 #L0 Length for u is 4 bits + End of table. Prefix code L0: + 000:e 0010:\\n 0110:! 0001:O 0101:b 0011:n 0111:s + 100:l 1010:" " 1110:K 1001:R 1101:k 1011:o 1111:u + 11,01 PFX IC0 is simple with 4 code words + 0012 2a |2Ah|10 IC0 ? bits: I5C4 + 0013 b5 ec 00|B5h IC0 ? bits: I6+xC7 + 0015 22 0010|111011 IC0 ? bits: I8+xC5 + 0016 8c 001100|0010 IC0 ? bits: I0C14+xx + 0 SHAPE False: lengths 2,2,2,2 + 0017 74 10,0|1 PFX D0 is simple with 3 code words + 0018 a6 0|01110 D0 1 bit: 2last-3 + 010011 D0 2 bits: 11xx-3 + 0019 aa 01010|1 D0 2 bits: 11xxx-3 + ====================Meta block contents===================== + |1,01 IC0 Literal: 9, copy: 5 + 001a 41 0001 0,0=L0 O + 100 0,48=L0 l + 001b a2 10|0 0,62=L0 l + 000 0,63=L0 e + 001c a1 1|101 0,59=L0 k + 000 0,63=L0 e + |1010 0,59=L0 " " + 001d b5 0101 0,11=L0 b + |1011 0,60=L0 o + 001e 24 0 0,3=D0 Distance: 2last-3=8 + Seen before: "lleke" + 0,10 IC0 Literal: 6, copy: 7 + |0010 0,59=L0 \\n + 001f 89 1001 0,7=L0 R + 000 0,52=L0 e + 0020 fa 010|1 0,58=L0 b + 1111 0,63=L0 u + 0021 eb 011|1 0,59=L0 s + 11,01 0,3=D0 Absolute value: 12 (pos 8) + Seen before: "olleke\\n" + 0022 db 01,1|1 IC0 Literal: 0, copy: 15 + |110,11 0,3=D0 Absolute value: 27 (pos 0) + Seen before: "Olleke bolleke\\n" + 0023 f8 00 IC0 Literal: 5, copy: 4 + 1110 0,7=L0 K + 0024 2c 00|11 0,52=L0 n + 1011 0,62=L0 o + 0025 0d 1|00 0,59=L0 l + 0110 0,63=L0 ! + """, + +'file': """ + >>> try: Layout(BitStream( + ... open("H:/Downloads/brotli-master/tests/testdata/10x10y.compressed",'rb') + ... .read())).processStream() + ... except NotImplementedError: pass + addr hex binary context explanation + -----------------------Stream header------------------------ + 0000 1b 1011 WSIZE windowsize=(1<<22)-16=4194288 + ======================Metablock header====================== + 1 LAST Last block: 1: True + 0 EMPTY Empty block: 0: False + 0001 13 00 |00h|13h,|00 MLEN data length: 0013h+1=20 + -------------------Block type descriptors------------------- + 0003 00 0 BT#L 0: 1 literal block type + 0 BT#I 0: 1 insert© block type + 0 BT#D 0: 1 distance block type + ------------------Distance code parameters------------------ + 0004 a4 0|000,00 DIST 0 postfix bits and 0000<<0=0 direct codes + -----------------------Context modes------------------------ + 10 LC0 Context mode for type 0: 2(UTF8) + ------------------------Context maps------------------------ + 0 LT# 0: 1 literal prefix tree + 0 DT# 0: 1 distance prefix tree + ---------------------Prefix code lists---------------------- + 0005 b0 0|1,01 PFX L0 is simple with 2 code words + 0006 b2 0|1011000 L0 1 bit: X + 0007 ea 0|1011001 L0 1 bit: Y + 01,01 PFX IC0 is simple with 2 code words + 0008 81 0000001|111 IC0 1 bit: I1C9&D=0 + 0009 47 02 0|47h|1 IC0 1 bit: I1C9 + 00,01 PFX D0 is simple with 1 code word + 000b 8a 010|000 D0 0 bits: 10x-3 + ====================Meta block contents===================== + 1 IC0 Literal: 1, copy: 9 + 0 0,0=L0 X + 0,() 0,3=D0 Absolute value: 1 (pos 0) + Seen before: "XXXXXXXXX" + 0 IC0 Literal: 1, copy: 9, same distance + |1 0,54=L0 Y + Seen before: "YYYYYYYYY" + """, + +'XY': """ + >>> try: Layout(BitStream(brotli.compress('X'*10+'Y'*10))).processStream() + ... except NotImplementedError: pass + addr hex binary context explanation + -----------------------Stream header------------------------ + 0000 1b 1011 WSIZE windowsize=(1<<22)-16=4194288 + ======================Metablock header====================== + 1 LAST Last block: 1: True + 0 EMPTY Empty block: 0: False + 0001 13 00 |00h|13h,|00 MLEN data length: 0013h+1=20 + -------------------Block type descriptors------------------- + 0003 00 0 BT#L 0: 1 literal block type + 0 BT#I 0: 1 insert© block type + 0 BT#D 0: 1 distance block type + ------------------Distance code parameters------------------ + 0004 a4 0|000,00 DIST 0 postfix bits and 0000<<0=0 direct codes + -----------------------Context modes------------------------ + 10 LC0 Context mode for type 0: 2(UTF8) + ------------------------Context maps------------------------ + 0 LT# 0: 1 literal prefix tree + 0 DT# 0: 1 distance prefix tree + ---------------------Prefix code lists---------------------- + 0005 b0 0|1,01 PFX L0 is simple with 2 code words + 0006 b2 0|1011000 L0 1 bit: X + 0007 82 0|1011001 L0 1 bit: Y + 00,01 PFX IC0 is simple with 1 code word + 0008 84 0000100|100 IC0 0 bits: I4C6&D=0 + 0009 00 00,0|1 PFX D0 is simple with 1 code word + 000a e0 0|00000 D0 0 bits: last + ====================Meta block contents===================== + () IC0 Literal: 4, copy: 6, same distance + 0 0,0=L0 X + 0 0,52=L0 X + 0 0,54=L0 X + 0 0,54=L0 X + Seen before: "XXXXXX" + () IC0 Literal: 4, copy: 6, same distance + 1 0,54=L0 Y + 1 0,54=L0 Y + |1 0,54=L0 Y + 000b 01 1 0,54=L0 Y + Seen before: "YYYYYY" + """, + +'empty': """ + >>> try: Layout(BitStream(b'\\x81\\x16\\x00\\x58')).processStream() + ... except NotImplementedError: pass + addr hex binary context explanation + -----------------------Stream header------------------------ + 0000 81 0000001 WSIZE windowsize=(1<<17)-16=131056 + ======================Metablock header====================== + |1 LAST Last block: 1: True + 0001 16 0 EMPTY Empty block: 0: False + 11 MLEN 11: empty block + 0 RSVD Reserved (must be zero) + 0002 00 000000|00,01 SKIP skip length: 0h+1=1 + |00 SKIP 2 bits ignored + Skipping to 4 + """, + +} + +if __name__=='__main__': + import sys + if len(sys.argv)>1: + l = Layout(BitStream(open(sys.argv[1],'rb').read())) + l.processStream() + else: + sys.path.append("h:/Persoonlijk/bin") + try: + import brotli + open('brotlidump.br', 'wb').write( + brotli.compress( + open('brotlidump.py', 'r').read() + )) + olleke = BitStream(brotli.compress( + 'Olleke bolleke\nRebusolleke\nOlleke bolleke\nKnol!')) + except ImportError: pass + import doctest + doctest.testmod(optionflags=doctest.REPORT_NDIFF + #|doctest.FAIL_FAST + ) diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/deorummolae.cc b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/deorummolae.cc new file mode 100644 index 000000000..d15b7ee55 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/deorummolae.cc @@ -0,0 +1,302 @@ +#include "./deorummolae.h" + +#include <array> +#include <cstdio> + +#include "./esaxx/sais.hxx" + +/* Used for quick SA-entry to file mapping. Each file is padded to size that + is a multiple of chunk size. */ +#define CHUNK_SIZE 64 +/* Length of substring that is considered to be covered by dictionary string. */ +#define CUT_MATCH 6 +/* Minimal dictionary entry size. */ +#define MIN_MATCH 24 + +/* Non tunable definitions. */ +#define CHUNK_MASK (CHUNK_SIZE - 1) +#define COVERAGE_SIZE (1 << (DM_LOG_MAX_FILES - 6)) + +/* File coverage: every bit set to 1 denotes a file covered by an isle. */ +typedef std::array<uint64_t, COVERAGE_SIZE> Coverage; + +/* Symbol of text alphabet. */ +typedef int32_t TextChar; + +/* Pointer to position in text. */ +typedef uint32_t TextIdx; + +/* SAIS sarray_type; unfortunately, must be a signed type. */ +typedef int32_t TextSaIdx; + +static size_t popcount(uint64_t u) { + return static_cast<size_t>(__builtin_popcountll(u)); +} + +/* Condense terminators and pad file entries. */ +static void rewriteText(std::vector<TextChar>* text) { + TextChar terminator = text->back(); + TextChar prev = terminator; + TextIdx to = 0; + for (TextIdx from = 0; from < text->size(); ++from) { + TextChar next = text->at(from); + if (next < 256 || prev < 256) { + text->at(to++) = next; + if (next >= 256) terminator = next; + } + prev = next; + } + text->resize(to); + if (text->empty()) text->push_back(terminator); + while (text->size() & CHUNK_MASK) text->push_back(terminator); +} + +/* Reenumerate terminators for smaller alphabet. */ +static void remapTerminators(std::vector<TextChar>* text, + TextChar* next_terminator) { + TextChar prev = -1; + TextChar x = 256; + for (TextIdx i = 0; i < text->size(); ++i) { + TextChar next = text->at(i); + if (next < 256) { // Char. + // Do nothing. + } else if (prev < 256) { // Terminator after char. + next = x++; + } else { // Terminator after terminator. + next = prev; + } + text->at(i) = next; + prev = next; + } + *next_terminator = x; +} + +/* Combine all file entries; create mapping position->file. */ +static void buildFullText(std::vector<std::vector<TextChar>>* data, + std::vector<TextChar>* full_text, std::vector<TextIdx>* file_map, + std::vector<TextIdx>* file_offset, TextChar* next_terminator) { + file_map->resize(0); + file_offset->resize(0); + full_text->resize(0); + for (TextIdx i = 0; i < data->size(); ++i) { + file_offset->push_back(full_text->size()); + std::vector<TextChar>& file = data->at(i); + rewriteText(&file); + full_text->insert(full_text->end(), file.begin(), file.end()); + file_map->insert(file_map->end(), file.size() / CHUNK_SIZE, i); + } + if (false) remapTerminators(full_text, next_terminator); +} + +/* Build longest-common-prefix based on suffix array and text. + TODO: borrowed -> unknown efficiency. */ +static void buildLcp(std::vector<TextChar>* text, std::vector<TextIdx>* sa, + std::vector<TextIdx>* lcp, std::vector<TextIdx>* invese_sa) { + TextIdx size = static_cast<TextIdx>(text->size()); + lcp->resize(size); + TextIdx k = 0; + lcp->at(size - 1) = 0; + for (TextIdx i = 0; i < size; ++i) { + if (invese_sa->at(i) == size - 1) { + k = 0; + continue; + } + // Suffix which follow i-th suffix. + TextIdx j = sa->at(invese_sa->at(i) + 1); + while (i + k < size && j + k < size && text->at(i + k) == text->at(j + k)) { + ++k; + } + lcp->at(invese_sa->at(i)) = k; + if (k > 0) --k; + } +} + +/* Isle is a range in SA with LCP not less than some value. + When we raise the LCP requirement, the isle sunks and smaller isles appear + instead. */ +typedef struct { + TextIdx lcp; + TextIdx l; + TextIdx r; + Coverage coverage; +} Isle; + +/* Helper routine for `cutMatch`. */ +static void poisonData(TextIdx pos, TextIdx length, + std::vector<std::vector<TextChar>>* data, std::vector<TextIdx>* file_map, + std::vector<TextIdx>* file_offset, TextChar* next_terminator) { + TextIdx f = file_map->at(pos / CHUNK_SIZE); + pos -= file_offset->at(f); + std::vector<TextChar>& file = data->at(f); + TextIdx l = (length == CUT_MATCH) ? CUT_MATCH : 1; + for (TextIdx j = 0; j < l; j++, pos++) { + if (file[pos] >= 256) continue; + if (file[pos + 1] >= 256) { + file[pos] = file[pos + 1]; + } else if (pos > 0 && file[pos - 1] >= 256) { + file[pos] = file[pos - 1]; + } else { + file[pos] = (*next_terminator)++; + } + } +} + +/* Remove substrings of a given match from files. + Substrings are replaced with unique terminators, so next iteration SA would + not allow to cross removed areas. */ +static void cutMatch(std::vector<std::vector<TextChar>>* data, TextIdx index, + TextIdx length, std::vector<TextIdx>* sa, std::vector<TextIdx>* lcp, + std::vector<TextIdx>* invese_sa, TextChar* next_terminator, + std::vector<TextIdx>* file_map, std::vector<TextIdx>* file_offset) { + while (length >= CUT_MATCH) { + TextIdx i = index; + while (lcp->at(i) >= length) { + i++; + poisonData( + sa->at(i), length, data, file_map, file_offset, next_terminator); + } + while (true) { + poisonData( + sa->at(index), length, data, file_map, file_offset, next_terminator); + if (index == 0 || lcp->at(index - 1) < length) break; + index--; + } + length--; + index = invese_sa->at(sa->at(index) + 1); + } +} + +std::string DM_generate(size_t dictionary_size_limit, + const std::vector<size_t>& sample_sizes, const uint8_t* sample_data) { + { + TextIdx tmp = static_cast<TextIdx>(dictionary_size_limit); + if ((tmp != dictionary_size_limit) || (tmp > 1u << 30)) { + fprintf(stderr, "dictionary_size_limit is too large\n"); + return ""; + } + } + + /* Could use 256 + '0' for easier debugging. */ + TextChar next_terminator = 256; + + std::string output; + std::vector<std::vector<TextChar>> data; + + TextIdx offset = 0; + size_t num_samples = sample_sizes.size(); + if (num_samples > DM_MAX_FILES) num_samples = DM_MAX_FILES; + for (size_t n = 0; n < num_samples; ++n) { + TextIdx delta = static_cast<TextIdx>(sample_sizes[n]); + if (delta != sample_sizes[n]) { + fprintf(stderr, "sample is too large\n"); + return ""; + } + if (delta == 0) { + fprintf(stderr, "0-length samples are prohibited\n"); + return ""; + } + TextIdx next_offset = offset + delta; + if (next_offset <= offset) { + fprintf(stderr, "corpus is too large\n"); + return ""; + } + data.push_back( + std::vector<TextChar>(sample_data + offset, sample_data + next_offset)); + offset = next_offset; + data.back().push_back(next_terminator++); + } + + /* Most arrays are allocated once, and then just resized to smaller and + smaller sizes. */ + std::vector<TextChar> full_text; + std::vector<TextIdx> file_map; + std::vector<TextIdx> file_offset; + std::vector<TextIdx> sa; + std::vector<TextIdx> invese_sa; + std::vector<TextIdx> lcp; + std::vector<Isle> isles; + std::vector<char> output_data; + TextIdx total = 0; + TextIdx total_cost = 0; + TextIdx best_cost; + Isle best_isle; + size_t min_count = num_samples; + + while (true) { + TextIdx max_match = static_cast<TextIdx>(dictionary_size_limit) - total; + buildFullText(&data, &full_text, &file_map, &file_offset, &next_terminator); + sa.resize(full_text.size()); + /* Hopefully, non-negative TextSaIdx is the same sa TextIdx counterpart. */ + saisxx(full_text.data(), reinterpret_cast<TextSaIdx*>(sa.data()), + static_cast<TextChar>(full_text.size()), next_terminator); + invese_sa.resize(full_text.size()); + for (TextIdx i = 0; i < full_text.size(); ++i) { + invese_sa[sa[i]] = i; + } + buildLcp(&full_text, &sa, &lcp, &invese_sa); + + /* Do not rebuild SA/LCP, just use different selection. */ + retry: + best_cost = 0; + best_isle = {0, 0, 0, {{0}}}; + isles.resize(0); + isles.push_back(best_isle); + + for (TextIdx i = 0; i < lcp.size(); ++i) { + TextIdx l = i; + Coverage cov = {{0}}; + size_t f = file_map[sa[i] / CHUNK_SIZE]; + cov[f >> 6] = (static_cast<uint64_t>(1)) << (f & 63); + while (lcp[i] < isles.back().lcp) { + Isle& top = isles.back(); + top.r = i; + l = top.l; + for (size_t x = 0; x < cov.size(); ++x) cov[x] |= top.coverage[x]; + size_t count = 0; + for (size_t x = 0; x < cov.size(); ++x) count += popcount(cov[x]); + TextIdx effective_lcp = top.lcp; + /* Restrict (last) dictionary entry length. */ + if (effective_lcp > max_match) effective_lcp = max_match; + TextIdx cost = count * effective_lcp; + if (cost > best_cost && count >= min_count && + effective_lcp >= MIN_MATCH) { + best_cost = cost; + best_isle = top; + best_isle.lcp = effective_lcp; + } + isles.pop_back(); + for (size_t x = 0; x < cov.size(); ++x) { + isles.back().coverage[x] |= cov[x]; + } + } + if (lcp[i] > isles.back().lcp) isles.push_back({lcp[i], l, 0, {{0}}}); + for (size_t x = 0; x < cov.size(); ++x) { + isles.back().coverage[x] |= cov[x]; + } + } + + /* When saturated matches do not match length restrictions, lower the + saturation requirements. */ + if (best_cost == 0 || best_isle.lcp < MIN_MATCH) { + if (min_count >= 8) { + min_count = (min_count * 7) / 8; + fprintf(stderr, "Retry: min_count=%zu\n", min_count); + goto retry; + } + break; + } + + /* Save the entry. */ + fprintf(stderr, "Savings: %d+%d, dictionary: %d+%d\n", + total_cost, best_cost, total, best_isle.lcp); + int* piece = &full_text[sa[best_isle.l]]; + output.insert(output.end(), piece, piece + best_isle.lcp); + total += best_isle.lcp; + total_cost += best_cost; + cutMatch(&data, best_isle.l, best_isle.lcp, &sa, &lcp, &invese_sa, + &next_terminator, &file_map, &file_offset); + if (total >= dictionary_size_limit) break; + } + + return output; +} diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/deorummolae.h b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/deorummolae.h new file mode 100644 index 000000000..581509726 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/deorummolae.h @@ -0,0 +1,26 @@ +#ifndef BROTLI_RESEARCH_DEORUMMOLAE_H_ +#define BROTLI_RESEARCH_DEORUMMOLAE_H_ + +#include <cstddef> +#include <cstdint> +#include <string> +#include <vector> + +/* log2(maximal number of files). Value 6 provides some speedups. */ +#define DM_LOG_MAX_FILES 6 + +/* Non tunable definitions. */ +#define DM_MAX_FILES (1 << DM_LOG_MAX_FILES) + +/** + * Generate a dictionary for given samples. + * + * @param dictionary_size_limit maximal dictionary size + * @param sample_sizes vector with sample sizes + * @param sample_data concatenated samples + * @return generated dictionary + */ +std::string DM_generate(size_t dictionary_size_limit, + const std::vector<size_t>& sample_sizes, const uint8_t* sample_data); + +#endif // BROTLI_RESEARCH_DEORUMMOLAE_H_ diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/dictionary_generator.cc b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/dictionary_generator.cc new file mode 100755 index 000000000..dcdf2fa12 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/dictionary_generator.cc @@ -0,0 +1,326 @@ +#include <climits> +#include <cstddef> +#include <cstdio> +#include <cstring> +#include <fstream> +#include <vector> + +#include "./deorummolae.h" +#include "./durchschlag.h" +#include "./sieve.h" + +/* This isn't a definitive list of "--foo" arguments, only those that take an + * additional "=#" integer parameter, like "--foo=20" or "--foo=32K". + */ +#define LONG_ARG_BLOCK_LEN "--block_len=" +#define LONG_ARG_SLICE_LEN "--slice_len=" +#define LONG_ARG_TARGET_DICT_LEN "--target_dict_len=" +#define LONG_ARG_MIN_SLICE_POP "--min_slice_pop=" +#define LONG_ARG_CHUNK_LEN "--chunk_len=" +#define LONG_ARG_OVERLAP_LEN "--overlap_len=" + +#define METHOD_DM 0 +#define METHOD_SIEVE 1 +#define METHOD_DURCHSCHLAG 2 +#define METHOD_DISTILL 3 +#define METHOD_PURIFY 4 + +static size_t readInt(const char* str) { + size_t result = 0; + if (str[0] == 0 || str[0] == '0') { + return 0; + } + for (size_t i = 0; i < 13; ++i) { + if (str[i] == 0) { + return result; + } + if (str[i] == 'k' || str[i] == 'K') { + if ((str[i + 1] == 0) && ((result << 10) > result)) { + return result << 10; + } + return 0; + } + if (str[i] == 'm' || str[i] == 'M') { + if ((str[i + 1] == 0) && ((result << 20) > result)) { + return result << 20; + } + return 0; + } + if (str[i] < '0' || str[i] > '9') { + return 0; + } + size_t next = (10 * result) + (str[i] - '0'); + if (next <= result) { + return 0; + } + result = next; + } + return 0; +} + +static std::string readFile(const std::string& path) { + std::ifstream file(path); + std::string content( + (std::istreambuf_iterator<char>(file)), std::istreambuf_iterator<char>()); + return content; +} + +static void writeFile(const char* file, const std::string& content) { + std::ofstream outfile(file, std::ofstream::binary); + outfile.write(content.c_str(), static_cast<std::streamsize>(content.size())); + outfile.close(); +} + +static void writeSamples(char const* argv[], const std::vector<int>& pathArgs, + const std::vector<size_t>& sizes, const uint8_t* data) { + size_t offset = 0; + for (size_t i = 0; i < pathArgs.size(); ++i) { + int j = pathArgs[i]; + const char* file = argv[j]; + size_t sampleSize = sizes[i]; + std::ofstream outfile(file, std::ofstream::binary); + outfile.write(reinterpret_cast<const char*>(data + offset), + static_cast<std::streamsize>(sampleSize)); + outfile.close(); + offset += sampleSize; + } +} + +/* Returns "base file name" or its tail, if it contains '/' or '\'. */ +static const char* fileName(const char* path) { + const char* separator_position = strrchr(path, '/'); + if (separator_position) path = separator_position + 1; + separator_position = strrchr(path, '\\'); + if (separator_position) path = separator_position + 1; + return path; +} + +static void printHelp(const char* name) { + fprintf(stderr, "Usage: %s [OPTION]... DICTIONARY [SAMPLE]...\n", name); + fprintf(stderr, + "Options:\n" + " --dm use 'deorummolae' engine\n" + " --distill rewrite samples; unique text parts are removed\n" + " --dsh use 'durchschlag' engine (default)\n" + " --purify rewrite samples; unique text parts are zeroed out\n" + " --sieve use 'sieve' engine\n" + " -b#, --block_len=#\n" + " set block length for 'durchschlag'; default: 1024\n" + " -s#, --slice_len=#\n" + " set slice length for 'distill', 'durchschlag', 'purify'\n" + " and 'sieve'; default: 16\n" + " -t#, --target_dict_len=#\n" + " set target dictionary length (limit); default: 16K\n" + " -u#, --min_slice_pop=#\n" + " set minimum slice population (for rewrites); default: 2\n" + " -c#, --chunk_len=#\n" + " if positive, samples are cut into chunks of this length;\n" + " default: 0; cannot mix with 'rewrite samples'\n" + " -o#, --overlap_len=#\n" + " set chunk overlap length; default 0\n" + "# is a decimal number with optional k/K/m/M suffix.\n" + "WARNING: 'distill' and 'purify' will overwrite original samples!\n" + " Completely unique samples might become empty files.\n\n"); +} + +int main(int argc, char const* argv[]) { + int dictionaryArg = -1; + int method = METHOD_DURCHSCHLAG; + size_t sliceLen = 16; + size_t targetSize = 16 << 10; + size_t blockSize = 1024; + size_t minimumPopulation = 2; + size_t chunkLen = 0; + size_t overlapLen = 0; + + std::vector<uint8_t> data; + std::vector<size_t> sizes; + std::vector<int> pathArgs; + size_t total = 0; + for (int i = 1; i < argc; ++i) { + if (argv[i] == nullptr) { + continue; + } + + if (argv[i][0] == '-') { + char arg1 = argv[i][1]; + const char* arg2 = arg1 ? &argv[i][2] : nullptr; + if (arg1 == '-') { + if (dictionaryArg != -1) { + fprintf(stderr, + "Method should be specified before dictionary / sample '%s'\n", + argv[i]); + exit(1); + } + + /* Look for "--long_arg" via exact match. */ + if (std::strcmp(argv[i], "--sieve") == 0) { + method = METHOD_SIEVE; + continue; + } + if (std::strcmp(argv[i], "--dm") == 0) { + method = METHOD_DM; + continue; + } + if (std::strcmp(argv[i], "--dsh") == 0) { + method = METHOD_DURCHSCHLAG; + continue; + } + if (std::strcmp(argv[i], "--distill") == 0) { + method = METHOD_DISTILL; + continue; + } + if (std::strcmp(argv[i], "--purify") == 0) { + method = METHOD_PURIFY; + continue; + } + + /* Look for "--long_arg=#" via prefix match. */ + if (std::strncmp(argv[i], LONG_ARG_BLOCK_LEN, + std::strlen(LONG_ARG_BLOCK_LEN)) == 0) { + arg1 = 'b'; + arg2 = &argv[i][std::strlen(LONG_ARG_BLOCK_LEN)]; + } else if (std::strncmp(argv[i], LONG_ARG_SLICE_LEN, + std::strlen(LONG_ARG_SLICE_LEN)) == 0) { + arg1 = 's'; + arg2 = &argv[i][std::strlen(LONG_ARG_SLICE_LEN)]; + } else if (std::strncmp(argv[i], LONG_ARG_TARGET_DICT_LEN, + std::strlen(LONG_ARG_TARGET_DICT_LEN)) == 0) { + arg1 = 't'; + arg2 = &argv[i][std::strlen(LONG_ARG_TARGET_DICT_LEN)]; + } else if (std::strncmp(argv[i], LONG_ARG_MIN_SLICE_POP, + std::strlen(LONG_ARG_MIN_SLICE_POP)) == 0) { + arg1 = 'u'; + arg2 = &argv[i][std::strlen(LONG_ARG_MIN_SLICE_POP)]; + } else if (std::strncmp(argv[i], LONG_ARG_CHUNK_LEN, + std::strlen(LONG_ARG_CHUNK_LEN)) == 0) { + arg1 = 'c'; + arg2 = &argv[i][std::strlen(LONG_ARG_CHUNK_LEN)]; + } else if (std::strncmp(argv[i], LONG_ARG_OVERLAP_LEN, + std::strlen(LONG_ARG_OVERLAP_LEN)) == 0) { + arg1 = 'o'; + arg2 = &argv[i][std::strlen(LONG_ARG_OVERLAP_LEN)]; + } else { + printHelp(fileName(argv[0])); + fprintf(stderr, "Invalid option '%s'\n", argv[i]); + exit(1); + } + } + + /* Look for "-f" short args or "--foo=#" long args. */ + if (arg1 == 'b') { + blockSize = readInt(arg2); + if (blockSize < 16 || blockSize > 65536) { + printHelp(fileName(argv[0])); + fprintf(stderr, "Invalid option '%s'\n", argv[i]); + exit(1); + } + } else if (arg1 == 's') { + sliceLen = readInt(arg2); + if (sliceLen < 4 || sliceLen > 256) { + printHelp(fileName(argv[0])); + fprintf(stderr, "Invalid option '%s'\n", argv[i]); + exit(1); + } + } else if (arg1 == 't') { + targetSize = readInt(arg2); + if (targetSize < 256 || targetSize > (1 << 25)) { + printHelp(fileName(argv[0])); + fprintf(stderr, "Invalid option '%s'\n", argv[i]); + exit(1); + } + } else if (arg1 == 'u') { + minimumPopulation = readInt(arg2); + if (minimumPopulation < 256 || minimumPopulation > 65536) { + printHelp(fileName(argv[0])); + fprintf(stderr, "Invalid option '%s'\n", argv[i]); + exit(1); + } + } else if (arg1 == 'c') { + chunkLen = readInt(arg2); + if (chunkLen < 0 || chunkLen > INT_MAX) { + printHelp(fileName(argv[0])); + fprintf(stderr, "Invalid option '%s'\n", argv[i]); + exit(1); + } + } else if (arg1 == 'o') { + overlapLen = readInt(arg2); + if (overlapLen < 0 || overlapLen > INT_MAX) { + printHelp(fileName(argv[0])); + fprintf(stderr, "Invalid option '%s'\n", argv[i]); + exit(1); + } + } else { + printHelp(fileName(argv[0])); + fprintf(stderr, "Unrecognized option '%s'\n", argv[i]); + exit(1); + } + continue; + } + + if (dictionaryArg == -1) { + if (method != METHOD_DISTILL && method != METHOD_PURIFY) { + dictionaryArg = i; + continue; + } + } + + std::string content = readFile(argv[i]); + if (chunkLen == 0) { + pathArgs.push_back(i); + data.insert(data.end(), content.begin(), content.end()); + total += content.size(); + sizes.push_back(content.size()); + continue; + } else if (chunkLen <= overlapLen) { + printHelp(fileName(argv[0])); + fprintf(stderr, "Invalid chunkLen - overlapLen combination\n"); + exit(1); + } + for (size_t chunkStart = 0; + chunkStart < content.size(); + chunkStart += chunkLen - overlapLen) { + std::string chunk = content.substr(chunkStart, chunkLen); + data.insert(data.end(), chunk.begin(), chunk.end()); + total += chunk.size(); + sizes.push_back(chunk.size()); + } + } + + bool wantDictionary = (dictionaryArg == -1); + if (method == METHOD_DISTILL || method == METHOD_PURIFY) { + wantDictionary = false; + if (chunkLen != 0) { + printHelp(fileName(argv[0])); + fprintf(stderr, "Cannot mix 'rewrite samples' with positive chunk_len\n"); + exit(1); + } + } + if (wantDictionary || total == 0) { + printHelp(fileName(argv[0])); + fprintf(stderr, "Not enough arguments\n"); + exit(1); + } + + if (method == METHOD_SIEVE) { + writeFile(argv[dictionaryArg], sieve_generate( + targetSize, sliceLen, sizes, data.data())); + } else if (method == METHOD_DM) { + writeFile(argv[dictionaryArg], DM_generate( + targetSize, sizes, data.data())); + } else if (method == METHOD_DURCHSCHLAG) { + writeFile(argv[dictionaryArg], durchschlag_generate( + targetSize, sliceLen, blockSize, sizes, data.data())); + } else if (method == METHOD_DISTILL) { + durchschlag_distill(sliceLen, minimumPopulation, &sizes, data.data()); + writeSamples(argv, pathArgs, sizes, data.data()); + } else if (method == METHOD_PURIFY) { + durchschlag_purify(sliceLen, minimumPopulation, sizes, data.data()); + writeSamples(argv, pathArgs, sizes, data.data()); + } else { + printHelp(fileName(argv[0])); + fprintf(stderr, "Unknown generator\n"); + exit(1); + } + return 0; +} diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/draw_diff.cc b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/draw_diff.cc new file mode 100644 index 000000000..1a5286960 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/draw_diff.cc @@ -0,0 +1,117 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + Author: zip753@gmail.com (Ivan Nikulin) + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Tool for drawing diff PPM images between two input PGM images. Normally used + with backward reference histogram drawing tool. */ + +#include <algorithm> +#include <cassert> +#include <cmath> +#include <cstdint> +#include <cstdio> +#include <cstdlib> /* exit, EXIT_FAILURE */ +#include <vector> + +#if !defined(CHECK) +#define CHECK(X) if (!(X)) exit(EXIT_FAILURE); +#endif + +typedef uint8_t* ScanLine; +typedef ScanLine* Image; + +void ReadPGM(FILE* f, Image* image, size_t* height, size_t* width) { + int colors; + CHECK(fscanf(f, "P5\n%lu %lu\n%d\n", width, height, &colors) == 3); + assert(colors == 255); + ScanLine* lines = new ScanLine[*height]; + *image = lines; + for (int i = *height - 1; i >= 0; --i) { + ScanLine line = new uint8_t[*width]; + lines[i] = line; + CHECK(fread(line, 1, *width, f) == *width); + } +} + +void CalculateDiff(int** diff, Image image1, Image image2, + size_t height, size_t width) { + for (size_t i = 0; i < height; ++i) { + for (size_t j = 0; j < width; ++j) { + diff[i][j] = static_cast<int>(image1[i][j]) - image2[i][j]; + } + } +} + +void DrawDiff(int** diff, Image image1, Image image2, + size_t height, size_t width, FILE* f) { + int max = -1234; + int min = +1234; + for (size_t i = 0; i < height; ++i) { + for (size_t j = 0; j < width; ++j) { + if (max < diff[i][j]) max = diff[i][j]; + if (min > diff[i][j]) min = diff[i][j]; + int img_min = std::min(255 - image1[i][j], 255 - image2[i][j]); + if (max < img_min) max = img_min; + } + } + + int abs_max = -min; + if (abs_max < max) abs_max = max; + + fprintf(f, "P6\n%lu %lu\n%d\n", width, height, abs_max); + + uint8_t* row = new uint8_t[3 * width]; + for (int i = height - 1; i >= 0; --i) { + for (int j = 0; j < width; ++j) { + int min_val = std::min(255 - image1[i][j], 255 - image2[i][j]); + int max_val = std::max(min_val, abs(diff[i][j])); + if (diff[i][j] > 0) { /* red */ + row[3 * j + 0] = abs_max - max_val + diff[i][j]; + row[3 * j + 1] = abs_max - max_val; + row[3 * j + 2] = abs_max - max_val + min_val; + } else { /* green */ + row[3 * j + 0] = abs_max - max_val; + row[3 * j + 1] = abs_max - max_val - diff[i][j]; + row[3 * j + 2] = abs_max - max_val + min_val; + } + } + fwrite(row, 1, 3 * width, f); + } + delete[] row; +} + +int main(int argc, char** argv) { + if (argc != 4) { + printf("usage: %s pgm1 pgm2 diff_ppm_path\n", argv[0]); + return 1; + } + + Image image1, image2; + size_t h1, w1, h2, w2; + + FILE* fimage1 = fopen(argv[1], "rb"); + ReadPGM(fimage1, &image1, &h1, &w1); + fclose(fimage1); + + FILE* fimage2 = fopen(argv[2], "rb"); + ReadPGM(fimage2, &image2, &h2, &w2); + fclose(fimage2); + + if (!(h1 == h2 && w1 == w2)) { + printf("Images must have the same size.\n"); + return 1; + } + + int** diff = new int*[h1]; + for (size_t i = 0; i < h1; ++i) diff[i] = new int[w1]; + CalculateDiff(diff, image1, image2, h1, w1); + + FILE* fdiff = fopen(argv[3], "wb"); + DrawDiff(diff, image1, image2, h1, w1, fdiff); + fclose(fdiff); + + return 0; +} diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/draw_histogram.cc b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/draw_histogram.cc new file mode 100644 index 000000000..b0192a217 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/draw_histogram.cc @@ -0,0 +1,197 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + Author: zip753@gmail.com (Ivan Nikulin) + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Backward reference visualization tool. Accepts file with backward references + as an input and produces PGM image with histogram of those references. */ + +#include <algorithm> /* min */ +#include <cassert> +#include <cstring> /* memset */ +#include <cmath> /* log, round */ +#include <cstdio> /* fscanf, fprintf */ +#include <cstdint> + +#include <gflags/gflags.h> +using gflags::ParseCommandLineFlags; + +#include "./read_dist.h" + +DEFINE_int32(height, 1000, "Height of the resulting histogam."); +DEFINE_int32(width, 8000, "Width of the resulting histogam."); +DEFINE_int32(size, 1e8, "Size of the compressed file."); +DEFINE_int32(brotli_window, -1, "Size of brotli window in bits."); +DEFINE_uint64(min_distance, 0, "Minimum distance."); +DEFINE_uint64(max_distance, 1 << 30, "Maximum distance."); +DEFINE_bool(with_copies, false, "True if input contains copy length."); +DEFINE_bool(simple, false, "True if using only black and white pixels."); +DEFINE_bool(linear, false, "True if using linear distance mapping."); +DEFINE_uint64(skip, 0, "Number of bytes to skip."); + +inline double DistanceTransform(double x) { + static bool linear = FLAGS_linear; + if (linear) { + return x; + } else { + /* Using log^2 scale because log scale produces big white gap at the bottom + of image. */ + return log(x) * log(x); + } +} + +/* Mapping pixel density on arc function to increase contrast. */ +inline double DensityTransform(double x) { + double z = 255 - x; + return sqrt(255 * 255 - z * z); +} + +inline int GetMaxDistance() { + return FLAGS_max_distance; +} + +void AdjustPosition(int* pos) { + static uint32_t offset = 0; + static int last = 0; + static uint32_t window_size = (1 << FLAGS_brotli_window); + assert(*pos >= 0 && *pos < window_size); + if (*pos < last) { + offset += window_size; + } + last = *pos; + *pos += offset; +} + +void BuildHistogram(FILE* fin, int** histo) { + int height = FLAGS_height; + int width = FLAGS_width; + int skip = FLAGS_skip; + size_t min_distance = FLAGS_min_distance; + + printf("height = %d, width = %d\n", height, width); + + for (int i = 0; i < height; i++) { + for (int j = 0; j < width; j++) { + histo[i][j] = 0; + } + } + + int max_pos = FLAGS_size - skip; + double min_dist = min_distance > 0 ? DistanceTransform(min_distance) : 0; + double max_dist = DistanceTransform(GetMaxDistance()) - min_dist; + int copy, pos, distance, x, y; + double dist; + while (ReadBackwardReference(fin, ©, &pos, &distance)) { + if (pos == -1) continue; // In case when only insert is present. + if (distance < min_distance || distance >= GetMaxDistance()) continue; + if (FLAGS_brotli_window != -1) { + AdjustPosition(&pos); + } + if (pos >= skip && distance <= pos) { + pos -= skip; + if (pos >= max_pos) break; + dist = DistanceTransform(static_cast<double>(distance)) - min_dist; + + x = std::min(static_cast<int>(round(dist / max_dist * height)), + height - 1); + y = 1ul * pos * width / max_pos; + if (!(y >= 0 && y < width)) { + printf("pos = %d, max_pos = %d, y = %d\n", pos, max_pos, y); + assert(y >= 0 && y < width); + } + + if (FLAGS_with_copies) { + int right = 1ul * (pos + copy - 1) * width / max_pos; + if (right < 0) { + printf("pos = %d, distance = %d, copy = %d, y = %d, right = %d\n", + pos, distance, copy, y, right); + assert(right >= 0); + } + if (y == right) { + histo[x][y] += copy; + } else { + int pos2 = static_cast<int>(ceil(1.0 * (y + 1) * max_pos / width)); + histo[x][y] += pos2 - pos; + for (int i = y + 1; i < right && i < width; ++i) { + histo[x][i] += max_pos / width; // Sometimes 1 more, but who cares. + } + // Make sure the match doesn't go beyond the image. + if (right < width) { + pos2 = static_cast<int>(ceil(1.0 * right * max_pos / width)); + histo[x][right] += pos + copy - 1 - pos2 + 1; + } + } + } else { + histo[x][y]++; + } + } + } +} + +void ConvertToPixels(int** histo, uint8_t** pixel) { + int height = FLAGS_height; + int width = FLAGS_width; + + int maxs = 0; + for (int i = 0; i < height; i++) { + for (int j = 0; j < width; j++) { + if (maxs < histo[i][j]) maxs = histo[i][j]; + } + } + + bool simple = FLAGS_simple; + double max_histo = static_cast<double>(maxs); + for (int i = 0; i < height; i++) { + for (int j = 0; j < width; j++) { + if (simple) { + pixel[i][j] = histo[i][j] > 0 ? 0 : 255; + } else { + pixel[i][j] = static_cast<uint8_t>( + 255 - DensityTransform(histo[i][j] / max_histo * 255)); + } + } + } +} + +void DrawPixels(uint8_t** pixel, FILE* fout) { + int height = FLAGS_height; + int width = FLAGS_width; + + fprintf(fout, "P5\n%d %d\n255\n", width, height); + for (int i = height - 1; i >= 0; i--) { + fwrite(pixel[i], 1, width, fout); + } +} + +int main(int argc, char* argv[]) { + ParseCommandLineFlags(&argc, &argv, true); + if (argc != 3) { + printf("usage: draw_histogram.cc data output_file\n"); + return 1; + } + + int height = FLAGS_height; + int width = FLAGS_width; + + FILE* fin = fopen(argv[1], "r"); + FILE* fout = fopen(argv[2], "wb"); + + uint8_t** pixel = new uint8_t*[height]; + int** histo = new int*[height]; + for (int i = 0; i < height; i++) { + pixel[i] = new uint8_t[width]; + histo[i] = new int[width]; + } + + BuildHistogram(fin, histo); + fclose(fin); + + ConvertToPixels(histo, pixel); + + DrawPixels(pixel, fout); + fclose(fout); + + return 0; +} diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/durchschlag.cc b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/durchschlag.cc new file mode 100755 index 000000000..2fbf41b58 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/durchschlag.cc @@ -0,0 +1,726 @@ +#include "./durchschlag.h" + +#include <algorithm> +#include <exception> /* terminate */ + +#include "divsufsort.h" + +/* Pointer to position in text. */ +typedef DurchschlagTextIdx TextIdx; + +/* (Sum of) value(s) of slice(s). */ +typedef uint32_t Score; + +typedef struct HashSlot { + TextIdx next; + TextIdx offset; +} HashSlot; + +typedef struct MetaSlot { + TextIdx mark; + Score score; +} MetaSlot; + +typedef struct Range { + TextIdx start; + TextIdx end; +} Range; + +typedef struct Candidate { + Score score; + TextIdx position; +} Candidate; + +struct greaterScore { + bool operator()(const Candidate& a, const Candidate& b) const { + return (a.score > b.score) || + ((a.score == b.score) && (a.position < b.position)); + } +}; + +struct lessScore { + bool operator()(const Candidate& a, const Candidate& b) const { + return (a.score < b.score) || + ((a.score == b.score) && (a.position > b.position)); + } +}; + +#define CANDIDATE_BUNDLE_SIZE (1 << 18) + +static void fatal(const char* error) { + fprintf(stderr, "%s\n", error); + std::terminate(); +} + +static TextIdx calculateDictionarySize(const std::vector<Range>& ranges) { + TextIdx result = 0; + for (size_t i = 0; i < ranges.size(); ++i) { + const Range& r = ranges[i]; + result += r.end - r.start; + } + return result; +} + +static std::string createDictionary( + const uint8_t* data, const std::vector<Range>& ranges, size_t limit) { + std::string output; + output.reserve(calculateDictionarySize(ranges)); + for (size_t i = 0; i < ranges.size(); ++i) { + const Range& r = ranges[i]; + output.insert(output.end(), &data[r.start], &data[r.end]); + } + if (output.size() > limit) { + output.resize(limit); + } + return output; +} + +/* precondition: span > 0 + precondition: end + span == len(shortcut) */ +static Score buildCandidatesList(std::vector<Candidate>* candidates, + std::vector<MetaSlot>* map, TextIdx span, const TextIdx* shortcut, + TextIdx end) { + candidates->resize(0); + + size_t n = map->size(); + MetaSlot* slots = map->data(); + for (size_t j = 0; j < n; ++j) { + slots[j].mark = 0; + } + + Score score = 0; + /* Consider the whole span, except one last item. The following loop will + add the last item to the end of the "chain", evaluate it, and cut one + "link" form the beginning. */ + for (size_t j = 0; j < span - 1; ++j) { + MetaSlot& item = slots[shortcut[j]]; + if (item.mark == 0) { + score += item.score; + } + item.mark++; + } + + TextIdx i = 0; + TextIdx limit = std::min<TextIdx>(end, CANDIDATE_BUNDLE_SIZE); + Score maxScore = 0; + for (; i < limit; ++i) { + TextIdx slice = shortcut[i + span - 1]; + MetaSlot& pick = slots[slice]; + if (pick.mark == 0) { + score += pick.score; + } + pick.mark++; + + if (score > maxScore) { + maxScore = score; + } + candidates->push_back({score, i}); + + MetaSlot& drop = slots[shortcut[i]]; + drop.mark--; + if (drop.mark == 0) { + score -= drop.score; + } + } + + std::make_heap(candidates->begin(), candidates->end(), greaterScore()); + Score minScore = candidates->at(0).score; + for (; i < end; ++i) { + TextIdx slice = shortcut[i + span - 1]; + MetaSlot& pick = slots[slice]; + if (pick.mark == 0) { + score += pick.score; + } + pick.mark++; + + if (score > maxScore) { + maxScore = score; + } + if (score >= minScore) { + candidates->push_back({score, i}); + std::push_heap(candidates->begin(), candidates->end(), greaterScore()); + if (candidates->size() > CANDIDATE_BUNDLE_SIZE && maxScore != minScore) { + while (candidates->at(0).score == minScore) { + std::pop_heap(candidates->begin(), candidates->end(), greaterScore()); + candidates->pop_back(); + } + minScore = candidates->at(0).score; + } + } + + MetaSlot& drop = slots[shortcut[i]]; + drop.mark--; + if (drop.mark == 0) { + score -= drop.score; + } + } + + for (size_t j = 0; j < n; ++j) { + slots[j].mark = 0; + } + + std::make_heap(candidates->begin(), candidates->end(), lessScore()); + return minScore; +} + +/* precondition: span > 0 + precondition: end + span == len(shortcut) */ +static Score rebuildCandidatesList(std::vector<TextIdx>* candidates, + std::vector<MetaSlot>* map, TextIdx span, const TextIdx* shortcut, + TextIdx end, TextIdx* next) { + size_t n = candidates->size(); + TextIdx* data = candidates->data(); + for (size_t i = 0; i < n; ++i) { + data[i] = 0; + } + + n = map->size(); + MetaSlot* slots = map->data(); + for (size_t i = 0; i < n; ++i) { + slots[i].mark = 0; + } + + Score score = 0; + /* Consider the whole span, except one last item. The following loop will + add the last item to the end of the "chain", evaluate it, and cut one + "link" form the beginning. */ + for (TextIdx i = 0; i < span - 1; ++i) { + MetaSlot& item = slots[shortcut[i]]; + if (item.mark == 0) { + score += item.score; + } + item.mark++; + } + + Score maxScore = 0; + for (TextIdx i = 0; i < end; ++i) { + MetaSlot& pick = slots[shortcut[i + span - 1]]; + if (pick.mark == 0) { + score += pick.score; + } + pick.mark++; + + if (candidates->size() <= score) { + candidates->resize(score + 1); + } + if (score > maxScore) { + maxScore = score; + } + next[i] = candidates->at(score); + candidates->at(score) = i; + + MetaSlot& drop = slots[shortcut[i]]; + drop.mark--; + if (drop.mark == 0) { + score -= drop.score; + } + } + + for (size_t i = 0; i < n; ++i) { + slots[i].mark = 0; + } + + candidates->resize(maxScore + 1); + return maxScore; +} + +static void addRange(std::vector<Range>* ranges, TextIdx start, TextIdx end) { + for (auto it = ranges->begin(); it != ranges->end();) { + if (end < it->start) { + ranges->insert(it, {start, end}); + return; + } + if (it->end < start) { + it++; + continue; + } + // Combine with existing. + start = std::min(start, it->start); + end = std::max(end, it->end); + // Remove consumed vector and continue. + it = ranges->erase(it); + } + ranges->push_back({start, end}); +} + +std::string durchschlag_generate( + size_t dictionary_size_limit, size_t slice_len, size_t block_len, + const std::vector<size_t>& sample_sizes, const uint8_t* sample_data) { + DurchschlagContext ctx = durchschlag_prepare( + slice_len, sample_sizes, sample_data); + return durchschlag_generate(DURCHSCHLAG_COLLABORATIVE, + dictionary_size_limit, block_len, ctx, sample_data); +} + +DurchschlagContext durchschlag_prepare(size_t slice_len, + const std::vector<size_t>& sample_sizes, const uint8_t* sample_data) { + /* Parameters aliasing */ + TextIdx sliceLen = static_cast<TextIdx>(slice_len); + if (sliceLen != slice_len) fatal("slice_len is too large"); + if (sliceLen < 1) fatal("slice_len is too small"); + const uint8_t* data = sample_data; + + TextIdx total = 0; + std::vector<TextIdx> offsets; + offsets.reserve(sample_sizes.size()); + for (size_t i = 0; i < sample_sizes.size(); ++i) { + TextIdx delta = static_cast<TextIdx>(sample_sizes[i]); + if (delta != sample_sizes[i]) fatal("sample is too large"); + if (delta == 0) fatal("0-length samples are prohibited"); + TextIdx next_total = total + delta; + if (next_total <= total) fatal("corpus is too large"); + total = next_total; + offsets.push_back(total); + } + + if (total < sliceLen) fatal("slice_len is larger than corpus size"); + TextIdx end = total - static_cast<TextIdx>(sliceLen) + 1; + TextIdx hashLen = 11; + while (hashLen < 29 && ((1u << hashLen) < end)) { + hashLen += 3; + } + hashLen -= 3; + TextIdx hashMask = (1u << hashLen) - 1u; + std::vector<TextIdx> hashHead(1 << hashLen); + TextIdx hash = 0; + TextIdx lShift = 3; + TextIdx rShift = hashLen - lShift; + for (TextIdx i = 0; i < sliceLen - 1; ++i) { + TextIdx v = data[i]; + hash = (((hash << lShift) | (hash >> rShift)) & hashMask) ^ v; + } + TextIdx lShiftX = (lShift * (sliceLen - 1)) % hashLen; + TextIdx rShiftX = hashLen - lShiftX; + + std::vector<HashSlot> map; + map.push_back({0, 0}); + TextIdx hashSlot = 1; + std::vector<TextIdx> sliceMap; + sliceMap.reserve(end); + for (TextIdx i = 0; i < end; ++i) { + TextIdx v = data[i + sliceLen - 1]; + TextIdx bucket = (((hash << lShift) | (hash >> rShift)) & hashMask) ^ v; + v = data[i]; + hash = bucket ^ (((v << lShiftX) | (v >> rShiftX)) & hashMask); + TextIdx slot = hashHead[bucket]; + while (slot != 0) { + HashSlot& item = map[slot]; + TextIdx start = item.offset; + bool miss = false; + for (TextIdx j = 0; j < sliceLen; ++j) { + if (data[i + j] != data[start + j]) { + miss = true; + break; + } + } + if (!miss) { + sliceMap.push_back(slot); + break; + } + slot = item.next; + } + if (slot == 0) { + map.push_back({hashHead[bucket], i}); + hashHead[bucket] = hashSlot; + sliceMap.push_back(hashSlot); + hashSlot++; + } + } + + return {total, sliceLen, static_cast<TextIdx>(map.size()), + std::move(offsets), std::move(sliceMap)}; +} + +DurchschlagContext durchschlag_prepare(size_t slice_len, + const std::vector<size_t>& sample_sizes, const DurchschlagIndex& index) { + /* Parameters aliasing */ + TextIdx sliceLen = static_cast<TextIdx>(slice_len); + if (sliceLen != slice_len) fatal("slice_len is too large"); + if (sliceLen < 1) fatal("slice_len is too small"); + const TextIdx* lcp = index.lcp.data(); + const TextIdx* sa = index.sa.data(); + + TextIdx total = 0; + std::vector<TextIdx> offsets; + offsets.reserve(sample_sizes.size()); + for (size_t i = 0; i < sample_sizes.size(); ++i) { + TextIdx delta = static_cast<TextIdx>(sample_sizes[i]); + if (delta != sample_sizes[i]) fatal("sample is too large"); + if (delta == 0) fatal("0-length samples are prohibited"); + TextIdx next_total = total + delta; + if (next_total <= total) fatal("corpus is too large"); + total = next_total; + offsets.push_back(total); + } + + if (total < sliceLen) fatal("slice_len is larger than corpus size"); + TextIdx counter = 1; + TextIdx end = total - sliceLen + 1; + std::vector<TextIdx> sliceMap(total); + TextIdx last = 0; + TextIdx current = 1; + while (current <= total) { + if (lcp[current - 1] < sliceLen) { + for (TextIdx i = last; i < current; ++i) { + sliceMap[sa[i]] = counter; + } + counter++; + last = current; + } + current++; + } + sliceMap.resize(end); + + // Reorder items for the better locality. + std::vector<TextIdx> reorder(counter); + counter = 1; + for (TextIdx i = 0; i < end; ++i) { + if (reorder[sliceMap[i]] == 0) { + reorder[sliceMap[i]] = counter++; + } + } + for (TextIdx i = 0; i < end; ++i) { + sliceMap[i] = reorder[sliceMap[i]]; + } + + return {total, sliceLen, counter, std::move(offsets), std::move(sliceMap)}; +} + +DurchschlagIndex durchschlag_index(const std::vector<uint8_t>& data) { + TextIdx total = static_cast<TextIdx>(data.size()); + if (total != data.size()) fatal("corpus is too large"); + saidx_t saTotal = static_cast<saidx_t>(total); + if (saTotal < 0) fatal("corpus is too large"); + if (static_cast<TextIdx>(saTotal) != total) fatal("corpus is too large"); + std::vector<TextIdx> sa(total); + /* Hopefully, non-negative int32_t values match TextIdx ones. */ + if (sizeof(TextIdx) != sizeof(int32_t)) fatal("type length mismatch"); + int32_t* saData = reinterpret_cast<int32_t*>(sa.data()); + divsufsort(data.data(), saData, saTotal); + + std::vector<TextIdx> isa(total); + for (TextIdx i = 0; i < total; ++i) isa[sa[i]] = i; + + // TODO: borrowed -> unknown efficiency. + std::vector<TextIdx> lcp(total); + TextIdx k = 0; + lcp[total - 1] = 0; + for (TextIdx i = 0; i < total; ++i) { + TextIdx current = isa[i]; + if (current == total - 1) { + k = 0; + continue; + } + TextIdx j = sa[current + 1]; // Suffix which follow i-th suffix. + while ((i + k < total) && (j + k < total) && (data[i + k] == data[j + k])) { + ++k; + } + lcp[current] = k; + if (k > 0) --k; + } + + return {std::move(lcp), std::move(sa)}; +} + +static void ScoreSlices(const std::vector<TextIdx>& offsets, + std::vector<MetaSlot>& map, const TextIdx* shortcut, TextIdx end) { + TextIdx piece = 0; + /* Fresh map contains all zeroes -> initial mark should be different. */ + TextIdx mark = 1; + for (TextIdx i = 0; i < end; ++i) { + if (offsets[piece] == i) { + piece++; + mark++; + } + MetaSlot& item = map[shortcut[i]]; + if (item.mark != mark) { + item.mark = mark; + item.score++; + } + } +} + +static std::string durchschlagGenerateExclusive( + size_t dictionary_size_limit, size_t block_len, + const DurchschlagContext& context, const uint8_t* sample_data) { + /* Parameters aliasing */ + TextIdx targetSize = static_cast<TextIdx>(dictionary_size_limit); + if (targetSize != dictionary_size_limit) { + fprintf(stderr, "dictionary_size_limit is too large\n"); + return ""; + } + TextIdx sliceLen = context.sliceLen; + TextIdx total = context.dataSize; + TextIdx blockLen = static_cast<TextIdx>(block_len); + if (blockLen != block_len) { + fprintf(stderr, "block_len is too large\n"); + return ""; + } + const uint8_t* data = sample_data; + const std::vector<TextIdx>& offsets = context.offsets; + std::vector<MetaSlot> map(context.numUniqueSlices); + const TextIdx* shortcut = context.sliceMap.data(); + + /* Initialization */ + if (blockLen < sliceLen) { + fprintf(stderr, "sliceLen is larger than block_len\n"); + return ""; + } + if (targetSize < blockLen || total < blockLen) { + fprintf(stderr, "block_len is too large\n"); + return ""; + } + TextIdx end = total - sliceLen + 1; + ScoreSlices(offsets, map, shortcut, end); + TextIdx span = blockLen - sliceLen + 1; + end = static_cast<TextIdx>(context.sliceMap.size()) - span; + std::vector<TextIdx> candidates; + std::vector<TextIdx> next(end); + Score maxScore = rebuildCandidatesList( + &candidates, &map, span, shortcut, end, next.data()); + + /* Block selection */ + const size_t triesLimit = (600 * 1000000) / span; + const size_t candidatesLimit = (150 * 1000000) / span; + std::vector<Range> ranges; + TextIdx mark = 0; + size_t numTries = 0; + while (true) { + TextIdx dictSize = calculateDictionarySize(ranges); + size_t numCandidates = 0; + if (dictSize > targetSize - blockLen) { + break; + } + if (maxScore == 0) { + break; + } + while (true) { + TextIdx candidate = 0; + while (maxScore > 0) { + if (candidates[maxScore] != 0) { + candidate = candidates[maxScore]; + candidates[maxScore] = next[candidate]; + break; + } + maxScore--; + } + if (maxScore == 0) { + break; + } + mark++; + numTries++; + numCandidates++; + Score score = 0; + for (size_t j = candidate; j < candidate + span; ++j) { + MetaSlot& item = map[shortcut[j]]; + if (item.mark != mark) { + score += item.score; + item.mark = mark; + } + } + if (score < maxScore) { + if (numTries < triesLimit && numCandidates < candidatesLimit) { + next[candidate] = candidates[score]; + candidates[score] = candidate; + } else { + maxScore = rebuildCandidatesList( + &candidates, &map, span, shortcut, end, next.data()); + mark = 0; + numTries = 0; + numCandidates = 0; + } + continue; + } else if (score > maxScore) { + fprintf(stderr, "Broken invariant\n"); + return ""; + } + for (TextIdx j = candidate; j < candidate + span; ++j) { + MetaSlot& item = map[shortcut[j]]; + item.score = 0; + } + addRange(&ranges, candidate, candidate + blockLen); + break; + } + } + + return createDictionary(data, ranges, targetSize); +} + +static std::string durchschlagGenerateCollaborative( + size_t dictionary_size_limit, size_t block_len, + const DurchschlagContext& context, const uint8_t* sample_data) { + /* Parameters aliasing */ + TextIdx targetSize = static_cast<TextIdx>(dictionary_size_limit); + if (targetSize != dictionary_size_limit) { + fprintf(stderr, "dictionary_size_limit is too large\n"); + return ""; + } + TextIdx sliceLen = context.sliceLen; + TextIdx total = context.dataSize; + TextIdx blockLen = static_cast<TextIdx>(block_len); + if (blockLen != block_len) { + fprintf(stderr, "block_len is too large\n"); + return ""; + } + const uint8_t* data = sample_data; + const std::vector<TextIdx>& offsets = context.offsets; + std::vector<MetaSlot> map(context.numUniqueSlices); + const TextIdx* shortcut = context.sliceMap.data(); + + /* Initialization */ + if (blockLen < sliceLen) { + fprintf(stderr, "sliceLen is larger than block_len\n"); + return ""; + } + if (targetSize < blockLen || total < blockLen) { + fprintf(stderr, "block_len is too large\n"); + return ""; + } + TextIdx end = total - sliceLen + 1; + ScoreSlices(offsets, map, shortcut, end); + TextIdx span = blockLen - sliceLen + 1; + end = static_cast<TextIdx>(context.sliceMap.size()) - span; + std::vector<Candidate> candidates; + candidates.reserve(CANDIDATE_BUNDLE_SIZE + 1024); + Score minScore = buildCandidatesList(&candidates, &map, span, shortcut, end); + + /* Block selection */ + std::vector<Range> ranges; + TextIdx mark = 0; + while (true) { + TextIdx dictSize = calculateDictionarySize(ranges); + if (dictSize > targetSize - blockLen) { + break; + } + if (minScore == 0 && candidates.empty()) { + break; + } + while (true) { + if (candidates.empty()) { + minScore = buildCandidatesList(&candidates, &map, span, shortcut, end); + mark = 0; + } + TextIdx candidate = candidates[0].position; + Score expectedScore = candidates[0].score; + if (expectedScore == 0) { + candidates.resize(0); + break; + } + std::pop_heap(candidates.begin(), candidates.end(), lessScore()); + candidates.pop_back(); + mark++; + Score score = 0; + for (TextIdx j = candidate; j < candidate + span; ++j) { + MetaSlot& item = map[shortcut[j]]; + if (item.mark != mark) { + score += item.score; + item.mark = mark; + } + } + if (score < expectedScore) { + if (score >= minScore) { + candidates.push_back({score, candidate}); + std::push_heap(candidates.begin(), candidates.end(), lessScore()); + } + continue; + } else if (score > expectedScore) { + fatal("Broken invariant"); + } + for (TextIdx j = candidate; j < candidate + span; ++j) { + MetaSlot& item = map[shortcut[j]]; + item.score = 0; + } + addRange(&ranges, candidate, candidate + blockLen); + break; + } + } + + return createDictionary(data, ranges, targetSize); +} + +std::string durchschlag_generate(DurchschalgResourceStrategy strategy, + size_t dictionary_size_limit, size_t block_len, + const DurchschlagContext& context, const uint8_t* sample_data) { + if (strategy == DURCHSCHLAG_COLLABORATIVE) { + return durchschlagGenerateCollaborative( + dictionary_size_limit, block_len, context, sample_data); + } else { + return durchschlagGenerateExclusive( + dictionary_size_limit, block_len, context, sample_data); + } +} + +void durchschlag_distill(size_t slice_len, size_t minimum_population, + std::vector<size_t>* sample_sizes, uint8_t* sample_data) { + /* Parameters aliasing */ + uint8_t* data = sample_data; + + /* Build slice map. */ + DurchschlagContext context = durchschlag_prepare( + slice_len, *sample_sizes, data); + + /* Calculate slice population. */ + const std::vector<TextIdx>& offsets = context.offsets; + std::vector<MetaSlot> map(context.numUniqueSlices); + const TextIdx* shortcut = context.sliceMap.data(); + TextIdx sliceLen = context.sliceLen; + TextIdx total = context.dataSize; + TextIdx end = total - sliceLen + 1; + ScoreSlices(offsets, map, shortcut, end); + + /* Condense samples, omitting unique slices. */ + TextIdx readPos = 0; + TextIdx writePos = 0; + TextIdx lastNonUniquePos = 0; + for (TextIdx i = 0; i < sample_sizes->size(); ++i) { + TextIdx sampleStart = writePos; + TextIdx oldSampleEnd = + readPos + static_cast<TextIdx>(sample_sizes->at(i)); + while (readPos < oldSampleEnd) { + if (readPos < end) { + MetaSlot& item = map[shortcut[readPos]]; + if (item.score >= minimum_population) { + lastNonUniquePos = readPos + sliceLen; + } + } + if (readPos < lastNonUniquePos) { + data[writePos++] = data[readPos]; + } + readPos++; + } + sample_sizes->at(i) = writePos - sampleStart; + } +} + +void durchschlag_purify(size_t slice_len, size_t minimum_population, + const std::vector<size_t>& sample_sizes, uint8_t* sample_data) { + /* Parameters aliasing */ + uint8_t* data = sample_data; + + /* Build slice map. */ + DurchschlagContext context = durchschlag_prepare( + slice_len, sample_sizes, data); + + /* Calculate slice population. */ + const std::vector<TextIdx>& offsets = context.offsets; + std::vector<MetaSlot> map(context.numUniqueSlices); + const TextIdx* shortcut = context.sliceMap.data(); + TextIdx sliceLen = context.sliceLen; + TextIdx total = context.dataSize; + TextIdx end = total - sliceLen + 1; + ScoreSlices(offsets, map, shortcut, end); + + /* Rewrite samples, zeroing out unique slices. */ + TextIdx lastNonUniquePos = 0; + for (TextIdx readPos = 0; readPos < total; ++readPos) { + if (readPos < end) { + MetaSlot& item = map[shortcut[readPos]]; + if (item.score >= minimum_population) { + lastNonUniquePos = readPos + sliceLen; + } + } + if (readPos >= lastNonUniquePos) { + data[readPos] = 0; + } + } +} diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/durchschlag.h b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/durchschlag.h new file mode 100755 index 000000000..adbc53141 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/durchschlag.h @@ -0,0 +1,99 @@ +#ifndef BROTLI_RESEARCH_DURCHSCHLAG_H_ +#define BROTLI_RESEARCH_DURCHSCHLAG_H_ + +#include <cstddef> +#include <cstdint> +#include <string> +#include <vector> + +/** + * Generate a dictionary for given samples. + * + * @param dictionary_size_limit maximal dictionary size + * @param slice_len text slice size + * @param block_len score block length + * @param sample_sizes vector with sample sizes + * @param sample_data concatenated samples + * @return generated dictionary + */ +std::string durchschlag_generate( + size_t dictionary_size_limit, size_t slice_len, size_t block_len, + const std::vector<size_t>& sample_sizes, const uint8_t* sample_data); + +//------------------------------------------------------------------------------ +// Lower level API for repetitive dictionary generation. +//------------------------------------------------------------------------------ + +/* Pointer to position in text. */ +typedef uint32_t DurchschlagTextIdx; + +/* Context is made public for flexible serialization / deserialization. */ +typedef struct DurchschlagContext { + DurchschlagTextIdx dataSize; + DurchschlagTextIdx sliceLen; + DurchschlagTextIdx numUniqueSlices; + std::vector<DurchschlagTextIdx> offsets; + std::vector<DurchschlagTextIdx> sliceMap; +} DurchschlagContext; + +DurchschlagContext durchschlag_prepare(size_t slice_len, + const std::vector<size_t>& sample_sizes, const uint8_t* sample_data); + +typedef enum DurchschalgResourceStrategy { + // Faster + DURCHSCHLAG_EXCLUSIVE = 0, + // Uses much less memory + DURCHSCHLAG_COLLABORATIVE = 1 +} DurchschalgResourceStrategy; + +std::string durchschlag_generate(DurchschalgResourceStrategy strategy, + size_t dictionary_size_limit, size_t block_len, + const DurchschlagContext& context, const uint8_t* sample_data); + +//------------------------------------------------------------------------------ +// Suffix Array based preparation. +//------------------------------------------------------------------------------ + +typedef struct DurchschlagIndex { + std::vector<DurchschlagTextIdx> lcp; + std::vector<DurchschlagTextIdx> sa; +} DurchschlagIndex; + +DurchschlagIndex durchschlag_index(const std::vector<uint8_t>& data); + +DurchschlagContext durchschlag_prepare(size_t slice_len, + const std::vector<size_t>& sample_sizes, const DurchschlagIndex& index); + +//------------------------------------------------------------------------------ +// Data preparation. +//------------------------------------------------------------------------------ + +/** + * Cut out unique slices. + * + * Both @p sample_sizes and @p sample_data are modified in-place. Number of + * samples remains unchanged, but some samples become shorter. + * + * @param slice_len (unique) slice size + * @param minimum_population minimum non-unique slice occurrence + * @param sample_sizes [in / out] vector with sample sizes + * @param sample_data [in / out] concatenated samples + */ +void durchschlag_distill(size_t slice_len, size_t minimum_population, + std::vector<size_t>* sample_sizes, uint8_t* sample_data); + +/** + * Replace unique slices with zeroes. + * + * @p sample_data is modified in-place. Number of samples and their length + * remain unchanged. + * + * @param slice_len (unique) slice size + * @param minimum_population minimum non-unique slice occurrence + * @param sample_sizes vector with sample sizes + * @param sample_data [in / out] concatenated samples + */ +void durchschlag_purify(size_t slice_len, size_t minimum_population, + const std::vector<size_t>& sample_sizes, uint8_t* sample_data); + +#endif // BROTLI_RESEARCH_DURCHSCHLAG_H_ diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/esaxx/COPYING b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/esaxx/COPYING new file mode 100644 index 000000000..07394df7c --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/esaxx/COPYING @@ -0,0 +1,24 @@ +This is the esaxx copyright. + +Copyright (c) 2010 Daisuke Okanohara All Rights Reserved. + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated documentation +files (the "Software"), to deal in the Software without +restriction, including without limitation the rights to use, +copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/esaxx/README b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/esaxx/README new file mode 100644 index 000000000..c3afa09de --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/esaxx/README @@ -0,0 +1,34 @@ +ESAXX +---------------------- + +This library provides the implementation of enhanced suffix array. +For an input text of length N, this library builds an enhanced suffix array in O(N) time +using 20N bytes. + +For a suffix array construction, I use sais.hxx, the induced sorting algorithm +implemented by Yuta Mori. + +It also provides the program to enumerate the statistics of all substrings in the text. + +> enum_substring + Enumerate all substring +> enum_substring -w + Input are words separated by space. + +Example: +------------------ +$ cat abra +abracadabra +$ enum_substring < abra + n:11 +alpha:256 + node:5 +0 2 4 abra +1 5 1 a +2 2 3 bra +3 2 2 ra +4 11 0 + +$ enum_substring -w < wiki.txt > + +Daisuke Okanohara <daisuke dot okanohara at gmail.com> diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/esaxx/cmdline.h b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/esaxx/cmdline.h new file mode 100644 index 000000000..2fc0260cb --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/esaxx/cmdline.h @@ -0,0 +1,704 @@ +/* +Copyright (c) 2009, Hideyuki Tanaka +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + * Neither the name of the <organization> nor the + names of its contributors may be used to endorse or promote products + derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY <copyright holder> ''AS IS'' AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL <copyright holder> BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#pragma once + +#include <iostream> +#include <sstream> +#include <vector> +#include <map> +#include <string> +#include <stdexcept> +#include <typeinfo> +#include <cstring> +#include <algorithm> +#include <cxxabi.h> + +namespace cmdline{ + +namespace detail{ + +template <typename Target, typename Source, bool Same> +class lexical_cast_t{ +public: + static Target cast(const Source &arg){ + Target ret; + std::stringstream ss; + if (!(ss<<arg && ss>>ret && ss.eof())) + throw std::bad_cast(); + + return ret; + } +}; + +template <typename Target, typename Source> +class lexical_cast_t<Target, Source, true>{ +public: + static Target cast(const Source &arg){ + return arg; + } +}; + +template <typename Source> +class lexical_cast_t<std::string, Source, false>{ +public: + static std::string cast(const Source &arg){ + std::ostringstream ss; + ss<<arg; + return ss.str(); + } +}; + +template <typename Target> +class lexical_cast_t<Target, std::string, false>{ +public: + static Target cast(const std::string &arg){ + Target ret; + std::istringstream ss(arg); + if (!(ss>>ret && ss.eof())) + throw std::bad_cast(); + return ret; + } +}; + +template <typename T1, typename T2> +struct is_same { + static const bool value = false; +}; + +template <typename T> +struct is_same<T, T>{ + static const bool value = true; +}; + +template<typename Target, typename Source> +Target lexical_cast(const Source &arg) +{ + return lexical_cast_t<Target, Source, detail::is_same<Target, Source>::value>::cast(arg); +} + +static inline std::string demangle(const std::string &name) +{ + int status=0; + char *p=abi::__cxa_demangle(name.c_str(), 0, 0, &status); + std::string ret(p); + free(p); + return ret; +} + +template <class T> +std::string readable_typename() +{ + return demangle(typeid(T).name()); +} + +template <> +std::string readable_typename<std::string>() +{ + return "string"; +} + +} // detail + +//----- + +class cmdline_error : public std::exception { +public: + cmdline_error(const std::string &msg): msg(msg){} + ~cmdline_error() throw() {} + const char *what() const throw() { return msg.c_str(); } +private: + std::string msg; +}; + +template <class T> +struct default_reader{ + T operator()(const std::string &str){ + return detail::lexical_cast<T>(str); + } +}; + +template <class T> +struct range_reader{ + range_reader(const T &low, const T &high): low(low), high(high) {} + T operator()(const std::string &s) const { + T ret=default_reader<T>()(s); + if (!(ret>=low && ret<=high)) throw cmdline::cmdline_error("range_error"); + return ret; + } +private: + T low, high; +}; + +template <class T> +range_reader<T> range(const T &low, const T &high) +{ + return range_reader<T>(low, high); +} + +template <class T> +struct oneof_reader{ + T operator()(const std::string &s){ + T ret=default_reader<T>()(s); + if (std::find(alt.begin(), alt.end(), s)==alt.end()) + throw cmdline_error(""); + return ret; + } + void add(const T &v){ alt.push_back(v); } +private: + std::vector<T> alt; +}; + +template <class T> +oneof_reader<T> oneof(T a1) +{ + oneof_reader<T> ret; + ret.add(a1); + return ret; +} + +template <class T> +oneof_reader<T> oneof(T a1, T a2) +{ + oneof_reader<T> ret; + ret.add(a1); + ret.add(a2); + return ret; +} + +template <class T> +oneof_reader<T> oneof(T a1, T a2, T a3) +{ + oneof_reader<T> ret; + ret.add(a1); + ret.add(a2); + ret.add(a3); + return ret; +} + +template <class T> +oneof_reader<T> oneof(T a1, T a2, T a3, T a4) +{ + oneof_reader<T> ret; + ret.add(a1); + ret.add(a2); + ret.add(a3); + ret.add(a4); + return ret; +} + +template <class T> +oneof_reader<T> oneof(T a1, T a2, T a3, T a4, T a5) +{ + oneof_reader<T> ret; + ret.add(a1); + ret.add(a2); + ret.add(a3); + ret.add(a4); + ret.add(a5); + return ret; +} + +template <class T> +oneof_reader<T> oneof(T a1, T a2, T a3, T a4, T a5, T a6) +{ + oneof_reader<T> ret; + ret.add(a1); + ret.add(a2); + ret.add(a3); + ret.add(a4); + ret.add(a5); + ret.add(a6); + return ret; +} + +template <class T> +oneof_reader<T> oneof(T a1, T a2, T a3, T a4, T a5, T a6, T a7) +{ + oneof_reader<T> ret; + ret.add(a1); + ret.add(a2); + ret.add(a3); + ret.add(a4); + ret.add(a5); + ret.add(a6); + ret.add(a7); + return ret; +} + +template <class T> +oneof_reader<T> oneof(T a1, T a2, T a3, T a4, T a5, T a6, T a7, T a8) +{ + oneof_reader<T> ret; + ret.add(a1); + ret.add(a2); + ret.add(a3); + ret.add(a4); + ret.add(a5); + ret.add(a6); + ret.add(a7); + ret.add(a8); + return ret; +} + +template <class T> +oneof_reader<T> oneof(T a1, T a2, T a3, T a4, T a5, T a6, T a7, T a8, T a9) +{ + oneof_reader<T> ret; + ret.add(a1); + ret.add(a2); + ret.add(a3); + ret.add(a4); + ret.add(a5); + ret.add(a6); + ret.add(a7); + ret.add(a8); + ret.add(a9); + return ret; +} + +template <class T> +oneof_reader<T> oneof(T a1, T a2, T a3, T a4, T a5, T a6, T a7, T a8, T a9, T a10) +{ + oneof_reader<T> ret; + ret.add(a1); + ret.add(a2); + ret.add(a3); + ret.add(a4); + ret.add(a5); + ret.add(a6); + ret.add(a7); + ret.add(a8); + ret.add(a9); + ret.add(a10); + return ret; +} + +//----- + +class parser{ +public: + parser(){ + } + ~parser(){ + for (std::map<std::string, option_base*>::iterator p=options.begin(); + p!=options.end(); p++) + delete p->second; + } + + void add(const std::string &name, + char short_name=0, + const std::string &desc=""){ + if (options.count(name)) throw cmdline_error("multiple definition: "+name); + options[name]=new option_without_value(name, short_name, desc); + ordered.push_back(options[name]); + } + + template <class T> + void add(const std::string &name, + char short_name=0, + const std::string &desc="", + bool need=true, + const T def=T()){ + add(name, short_name, desc, need, def, default_reader<T>()); + } + + template <class T, class F> + void add(const std::string &name, + char short_name=0, + const std::string &desc="", + bool need=true, + const T def=T(), + F reader=F()){ + if (options.count(name)) throw cmdline_error("multiple definition: "+name); + options[name]=new option_with_value_with_reader<T, F>(name, short_name, need, def, desc, reader); + ordered.push_back(options[name]); + } + + void footer(const std::string &f){ + ftr=f; + } + + void set_program_name(const std::string &name){ + prog_name=name; + } + + bool exist(const std::string &name){ + if (options.count(name)==0) throw cmdline_error("there is no flag: --"+name); + return options[name]->has_set(); + } + + template <class T> + const T &get(const std::string &name) const { + if (options.count(name)==0) throw cmdline_error("there is no flag: --"+name); + const option_with_value<T> *p=dynamic_cast<const option_with_value<T>*>(options.find(name)->second); + if (p==NULL) throw cmdline_error("type mismatch flag '"+name+"'"); + return p->get(); + } + + const std::vector<std::string> &rest() const { + return others; + } + + bool parse(int argc, char *argv[]){ + errors.clear(); + others.clear(); + + if (argc<1){ + errors.push_back("argument number must be longer than 0"); + return false; + } + if (prog_name=="") + prog_name=argv[0]; + + std::map<char, std::string> lookup; + for (std::map<std::string, option_base*>::iterator p=options.begin(); + p!=options.end(); p++){ + if (p->first.length()==0) continue; + char initial=p->second->short_name(); + if (initial){ + if (lookup.count(initial)>0){ + lookup[initial]=""; + errors.push_back(std::string("short option '")+initial+"' is ambiguous"); + return false; + } + else lookup[initial]=p->first; + } + } + + for (int i=1; i<argc; i++){ + if (strncmp(argv[i], "--", 2)==0){ + char *p=strchr(argv[i]+2, '='); + if (p){ + std::string name(argv[i]+2, p); + std::string val(p+1); + set_option(name, val); + } + else{ + std::string name(argv[i]+2); + set_option(name); + } + } + else if (strncmp(argv[i], "-", 1)==0){ + if (!argv[i][1]) continue; + char last=argv[i][1]; + for (int j=2; argv[i][j]; j++){ + last=argv[i][j]; + if (lookup.count(argv[i][j-1])==0){ + errors.push_back(std::string("undefined short option: -")+argv[i][j-1]); + continue; + } + if (lookup[argv[i][j-1]]==""){ + errors.push_back(std::string("ambiguous short option: -")+argv[i][j-1]); + continue; + } + set_option(lookup[argv[i][j-1]]); + } + + if (lookup.count(last)==0){ + errors.push_back(std::string("undefined short option: -")+last); + continue; + } + if (lookup[last]==""){ + errors.push_back(std::string("ambiguous short option: -")+last); + continue; + } + + if (i+1<argc && options[lookup[last]]->has_value()){ + set_option(lookup[last], argv[i+1]); + i++; + } + else{ + set_option(lookup[last]); + } + } + else{ + others.push_back(argv[i]); + } + } + + for (std::map<std::string, option_base*>::iterator p=options.begin(); + p!=options.end(); p++) + if (!p->second->valid()) + errors.push_back("need option: --"+std::string(p->first)); + + return errors.size()==0; + } + + std::string error() const{ + return errors.size()>0?errors[0]:""; + } + + std::string error_full() const{ + std::ostringstream oss; + for (size_t i=0; i<errors.size(); i++) + oss<<errors[i]<<std::endl; + return oss.str(); + } + + std::string usage() const { + std::ostringstream oss; + oss<<"usage: "<<prog_name<<" "; + for (size_t i=0; i<ordered.size(); i++){ + if (ordered[i]->must()) + oss<<ordered[i]->short_description()<<" "; + } + + oss<<"[options] ... "<<ftr<<std::endl; + oss<<"options:"<<std::endl; + + size_t max_width=0; + for (size_t i=0; i<ordered.size(); i++){ + max_width=std::max(max_width, ordered[i]->name().length()); + } + for (size_t i=0; i<ordered.size(); i++){ + if (ordered[i]->short_name()){ + oss<<" -"<<ordered[i]->short_name()<<", "; + } + else{ + oss<<" "; + } + + oss<<"--"<<ordered[i]->name(); + for (size_t j=ordered[i]->name().length(); j<max_width+4; j++) + oss<<' '; + oss<<ordered[i]->description()<<std::endl; + } + return oss.str(); + } + +private: + + void set_option(const std::string &name){ + if (options.count(name)==0){ + errors.push_back("undefined option: --"+name); + return; + } + if (!options[name]->set()){ + errors.push_back("option needs value: --"+name); + return; + } + } + + void set_option(const std::string &name, const std::string &value){ + if (options.count(name)==0){ + errors.push_back("undefined option: --"+name); + return; + } + if (!options[name]->set(value)){ + errors.push_back("option value is invalid: --"+name+"="+value); + return; + } + } + + class option_base{ + public: + virtual ~option_base(){} + + virtual bool has_value() const=0; + virtual bool set()=0; + virtual bool set(const std::string &value)=0; + virtual bool has_set() const=0; + virtual bool valid() const=0; + virtual bool must() const=0; + + virtual const std::string &name() const=0; + virtual char short_name() const=0; + virtual const std::string &description() const=0; + virtual std::string short_description() const=0; + }; + + class option_without_value : public option_base { + public: + option_without_value(const std::string &name, + char short_name, + const std::string &desc) + :nam(name), snam(short_name), desc(desc), has(false){ + } + ~option_without_value(){} + + bool has_value() const { return false; } + + bool set(){ + has=true; + return true; + } + + bool set(const std::string &){ + return false; + } + + bool has_set() const { + return has; + } + + bool valid() const{ + return true; + } + + bool must() const{ + return false; + } + + const std::string &name() const{ + return nam; + } + + char short_name() const{ + return snam; + } + + const std::string &description() const { + return desc; + } + + std::string short_description() const{ + return "--"+nam; + } + + private: + std::string nam; + char snam; + std::string desc; + bool has; + }; + + template <class T> + class option_with_value : public option_base { + public: + option_with_value(const std::string &name, + char short_name, + bool need, + const T &def, + const std::string &desc) + : nam(name), snam(short_name), need(need), has(false) + , def(def), actual(def) { + this->desc=full_description(desc); + } + ~option_with_value(){} + + const T &get() const { + return actual; + } + + bool has_value() const { return true; } + + bool set(){ + return false; + } + + bool set(const std::string &value){ + try{ + actual=read(value); + has=true; + } + catch(const std::exception &e){ + return false; + } + return true; + } + + bool has_set() const{ + return has; + } + + bool valid() const{ + if (need && !has) return false; + return true; + } + + bool must() const{ + return need; + } + + const std::string &name() const{ + return nam; + } + + char short_name() const{ + return snam; + } + + const std::string &description() const { + return desc; + } + + std::string short_description() const{ + return "--"+nam+"="+detail::readable_typename<T>(); + } + + protected: + std::string full_description(const std::string &desc){ + return + desc+" ("+detail::readable_typename<T>()+ + (need?"":" [="+detail::lexical_cast<std::string>(def)+"]") + +")"; + } + + virtual T read(const std::string &s)=0; + + std::string nam; + char snam; + bool need; + std::string desc; + + bool has; + T def; + T actual; + }; + + template <class T, class F> + class option_with_value_with_reader : public option_with_value<T> { + public: + option_with_value_with_reader(const std::string &name, + char short_name, + bool need, + const T def, + const std::string &desc, + F reader) + : option_with_value<T>(name, short_name, need, def, desc), reader(reader){ + } + + private: + T read(const std::string &s){ + return reader(s); + } + + F reader; + }; + + std::map<std::string, option_base*> options; + std::vector<option_base*> ordered; + std::string ftr; + + std::string prog_name; + std::vector<std::string> others; + + std::vector<std::string> errors; +}; + +} // cmdline diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/esaxx/enumSubstring.cpp b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/esaxx/enumSubstring.cpp new file mode 100644 index 000000000..e34842fd2 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/esaxx/enumSubstring.cpp @@ -0,0 +1,140 @@ +#include <iostream> +#include <string> +#include <vector> +#include <map> +#include "cmdline.h" +#include "esa.hxx" + +using namespace std; + +int readFile(const char* fn, vector<int>& T){ + FILE* fp = fopen(fn, "rb"); + if (fp == NULL){ + cerr << "cannot open " << fn << endl; + return -1; + } + + if (fseek(fp, 0, SEEK_END) != 0){ + cerr << "cannot fseek " << fn << endl; + fclose(fp); + return -1; + } + int n = ftell(fp); + rewind(fp); + if (n < 0){ + cerr << "cannot ftell " << fn << endl; + fclose(fp); + return -1; + } + T.resize(n); + if (fread(&T[0], sizeof(unsigned char), (size_t)n, fp) != (size_t) n){ + cerr << "fread error " << fn << endl; + fclose(fp); + return -1; + } + + fclose(fp); + return 0; +} + +int getID(const string& str, map<string, int>& word2id){ + map<string, int>::const_iterator it = word2id.find(str); + if (it == word2id.end()){ + int newID = (int)word2id.size(); + word2id[str] = newID; + return newID; + } else { + return it->second; + } +} + +void printSnipet(const vector<int>& T, const int beg, const int len, const vector<string>& id2word){ + for (int i = 0; i < len; ++i){ + int c = T[beg + i]; + if (id2word.size() > 0){ + cout << id2word[c] << " "; + } else { + cout << (isspace((char)c) ? '_' : (char)c); + } + } +} + +int main(int argc, char* argv[]){ + cmdline::parser p; + p.add("word", 'w', "word type"); + + if (!p.parse(argc, argv)){ + cerr << p.error() << endl + << p.usage() << endl; + return -1; + } + + if (p.rest().size() > 0){ + cerr << p.usage() << endl; + return -1; + } + + vector<int> T; + + bool isWord = p.exist("word"); + map<string, int> word2id; + istreambuf_iterator<char> isit(cin); + istreambuf_iterator<char> end; + + size_t origLen = 0; + if (isWord){ + string word; + while (isit != end){ + char c = *isit++; + if (!isspace(c)){ + word += c; + } else if (word.size() > 0){ + T.push_back(getID(word, word2id)); + word = ""; + } + ++origLen; + } + if (word.size() > 0){ + T.push_back(getID(word, word2id)); + } + } else { + while (isit != end){ + T.push_back((unsigned char)(*isit++)); + } + origLen = T.size(); + } + + vector<string> id2word(word2id.size()); + for (map<string, int>::const_iterator it = word2id.begin(); + it != word2id.end(); ++it){ + id2word[it->second] = it->first; + } + + vector<int> SA(T.size()); + vector<int> L (T.size()); + vector<int> R (T.size()); + vector<int> D (T.size()); + + int k = (isWord) ? (int)id2word.size() : 0x100; + if (isWord){ + cerr << "origN:" << origLen << endl; + } + cerr << " n:" << T.size() << endl; + cerr << "alpha:" << k << endl; + + int nodeNum = 0; + if (esaxx(T.begin(), SA.begin(), + L.begin(), R.begin(), D.begin(), + (int)T.size(), k, nodeNum) == -1){ + return -1; + } + cerr << " node:" << nodeNum << endl; + + for (int i = 0; i < nodeNum; ++i){ + cout << i << "\t" << R[i] - L[i] << "\t" << D[i] << "\t"; + printSnipet(T, SA[L[i]], D[i], id2word); + cout << endl; + } + + return 0; +} diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/esaxx/esa.hxx b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/esaxx/esa.hxx new file mode 100644 index 000000000..acb5c7a1b --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/esaxx/esa.hxx @@ -0,0 +1,125 @@ +/* + * esa.hxx + * Copyright (c) 2010 Daisuke Okanohara All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef _ESA_HXX +#define _ESA_HXX + +#include <vector> +#include <utility> +#include <cassert> +#include "sais.hxx" + +namespace esaxx_private { +template<typename string_type, typename sarray_type, typename index_type> +index_type suffixtree(string_type T, sarray_type SA, sarray_type L, sarray_type R, sarray_type D, index_type n){ + if (n == 0){ + return 0; + } + sarray_type Psi = L; + Psi[SA[0]] = SA[n-1]; + for (index_type i = 1; i < n; ++i){ + Psi[SA[i]] = SA[i-1]; + } + + // Compare at most 2n log n charcters. Practically fastest + // "Permuted Longest-Common-Prefix Array", Juha Karkkainen, CPM 09 + sarray_type PLCP = R; + index_type h = 0; + for (index_type i = 0; i < n; ++i){ + index_type j = Psi[i]; + while (i+h < n && j+h < n && + T[i+h] == T[j+h]){ + ++h; + } + PLCP[i] = h; + if (h > 0) --h; + } + + sarray_type H = L; + for (index_type i = 0; i < n; ++i){ + H[i] = PLCP[SA[i]]; + } + H[0] = -1; + + std::vector<std::pair<index_type, index_type> > S; + S.push_back(std::make_pair((index_type)-1, (index_type)-1)); + size_t nodeNum = 0; + for (index_type i = 0; ; ++i){ + std::pair<index_type, index_type> cur (i, (i == n) ? -1 : H[i]); + std::pair<index_type, index_type> cand(S.back()); + while (cand.second > cur.second){ + if (i - cand.first > 1){ + L[nodeNum] = cand.first; + R[nodeNum] = i; + D[nodeNum] = cand.second; + ++nodeNum; + } + cur.first = cand.first; + S.pop_back(); + cand = S.back(); + } + if (cand.second < cur.second){ + S.push_back(cur); + } + if (i == n) break; + S.push_back(std::make_pair(i, n - SA[i] + 1)); + } + return nodeNum; +} +} + +/** + * @brief Build an enhanced suffix array of a given string in linear time + * For an input text T, esaxx() builds an enhancd suffix array in linear time. + * i-th internal node is represented as a triple (L[i], R[i], D[i]); + * L[i] and R[i] is the left/right boundary of the suffix array as SA[L[i]....R[i]-1] + * D[i] is the depth of the internal node + * The number of internal node is at most N-1 and return the actual number by + * @param T[0...n-1] The input string. (random access iterator) + * @param SA[0...n-1] The output suffix array (random access iterator) + * @param L[0...n-1] The output left boundary of internal node (random access iterator) + * @param R[0...n-1] The output right boundary of internal node (random access iterator) + * @param D[0...n-1] The output depth of internal node (random access iterator) + * @param n The length of the input string + * @param k The alphabet size + * @pram nodeNum The output the number of internal node + * @return 0 if succeded, -1 or -2 otherwise + */ + +template<typename string_type, typename sarray_type, typename index_type> +int esaxx(string_type T, sarray_type SA, sarray_type L, sarray_type R, sarray_type D, + index_type n, index_type k, index_type& nodeNum) { + if ((n < 0) || (k <= 0)) return -1; + int err = saisxx(T, SA, n, k); + if (err != 0){ + return err; + } + nodeNum = esaxx_private::suffixtree(T, SA, L, R, D, n); + return 0; +} + + +#endif // _ESA_HXX diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/esaxx/sais.hxx b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/esaxx/sais.hxx new file mode 100644 index 000000000..20e69dfec --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/esaxx/sais.hxx @@ -0,0 +1,364 @@ +/* + * sais.hxx for sais-lite + * Copyright (c) 2008-2009 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef _SAIS_HXX +#define _SAIS_HXX 1 +#ifdef __cplusplus + +#ifdef __INTEL_COMPILER +#pragma warning(disable : 383 981 1418) +// for icc 64-bit +//#define __builtin_vsnprintf(a, b, c, d) __builtin_vsnprintf(a, b, c, (char *)d) +#endif + +#include <iterator> +#ifdef _OPENMP +# include <omp.h> +#endif + +namespace saisxx_private { + +/* find the start or end of each bucket */ +template<typename string_type, typename bucket_type, typename index_type> +void +getCounts(const string_type T, bucket_type C, index_type n, index_type k) { +#ifdef _OPENMP + bucket_type D; + index_type i, j, p, sum, first, last; + int thnum, maxthreads = omp_get_max_threads(); +#pragma omp parallel default(shared) private(D, i, thnum, first, last) + { + thnum = omp_get_thread_num(); + D = C + thnum * k; + first = n / maxthreads * thnum; + last = (thnum < (maxthreads - 1)) ? n / maxthreads * (thnum + 1) : n; + for(i = 0; i < k; ++i) { D[i] = 0; } + for(i = first; i < last; ++i) { ++D[T[i]]; } + } + if(1 < maxthreads) { +#pragma omp parallel for default(shared) private(i, j, p, sum) + for(i = 0; i < k; ++i) { + for(j = 1, p = i + k, sum = C[i]; j < maxthreads; ++j, p += k) { + sum += C[p]; + } + C[i] = sum; + } + } +#else + index_type i; + for(i = 0; i < k; ++i) { C[i] = 0; } + for(i = 0; i < n; ++i) { ++C[T[i]]; } +#endif +} +template<typename bucket_type, typename index_type> +void +getBuckets(const bucket_type C, bucket_type B, index_type k, bool end) { + index_type i, sum = 0; + if(end) { for(i = 0; i < k; ++i) { sum += C[i]; B[i] = sum; } } + else { for(i = 0; i < k; ++i) { sum += C[i]; B[i] = sum - C[i]; } } +} + +/* compute SA and BWT */ +template<typename string_type, typename sarray_type, + typename bucket_type, typename index_type> +void +induceSA(string_type T, sarray_type SA, bucket_type C, bucket_type B, + index_type n, index_type k) { +typedef typename std::iterator_traits<string_type>::value_type char_type; + sarray_type b; + index_type i, j; + char_type c0, c1; + /* compute SAl */ + if(C == B) { getCounts(T, C, n, k); } + getBuckets(C, B, k, false); /* find starts of buckets */ + b = SA + B[c1 = T[j = n - 1]]; + *b++ = ((0 < j) && (T[j - 1] < c1)) ? ~j : j; + for(i = 0; i < n; ++i) { + j = SA[i], SA[i] = ~j; + if(0 < j) { + if((c0 = T[--j]) != c1) { B[c1] = b - SA; b = SA + B[c1 = c0]; } + *b++ = ((0 < j) && (T[j - 1] < c1)) ? ~j : j; + } + } + /* compute SAs */ + if(C == B) { getCounts(T, C, n, k); } + getBuckets(C, B, k, true); /* find ends of buckets */ + for(i = n - 1, b = SA + B[c1 = 0]; 0 <= i; --i) { + if(0 < (j = SA[i])) { + if((c0 = T[--j]) != c1) { B[c1] = b - SA; b = SA + B[c1 = c0]; } + *--b = ((j == 0) || (T[j - 1] > c1)) ? ~j : j; + } else { + SA[i] = ~j; + } + } +} +template<typename string_type, typename sarray_type, + typename bucket_type, typename index_type> +int +computeBWT(string_type T, sarray_type SA, bucket_type C, bucket_type B, + index_type n, index_type k) { +typedef typename std::iterator_traits<string_type>::value_type char_type; + sarray_type b; + index_type i, j, pidx = -1; + char_type c0, c1; + /* compute SAl */ + if(C == B) { getCounts(T, C, n, k); } + getBuckets(C, B, k, false); /* find starts of buckets */ + b = SA + B[c1 = T[j = n - 1]]; + *b++ = ((0 < j) && (T[j - 1] < c1)) ? ~j : j; + for(i = 0; i < n; ++i) { + if(0 < (j = SA[i])) { + SA[i] = ~(c0 = T[--j]); + if(c0 != c1) { B[c1] = b - SA; b = SA + B[c1 = c0]; } + *b++ = ((0 < j) && (T[j - 1] < c1)) ? ~j : j; + } else if(j != 0) { + SA[i] = ~j; + } + } + /* compute SAs */ + if(C == B) { getCounts(T, C, n, k); } + getBuckets(C, B, k, true); /* find ends of buckets */ + for(i = n - 1, b = SA + B[c1 = 0]; 0 <= i; --i) { + if(0 < (j = SA[i])) { + SA[i] = (c0 = T[--j]); + if(c0 != c1) { B[c1] = b - SA; b = SA + B[c1 = c0]; } + *--b = ((0 < j) && (T[j - 1] > c1)) ? ~((index_type)T[j - 1]) : j; + } else if(j != 0) { + SA[i] = ~j; + } else { + pidx = i; + } + } + return pidx; +} + +/* find the suffix array SA of T[0..n-1] in {0..k}^n + use a working space (excluding s and SA) of at most 2n+O(1) for a constant alphabet */ +template<typename string_type, typename sarray_type, typename index_type> +int +suffixsort(string_type T, sarray_type SA, + index_type fs, index_type n, index_type k, + bool isbwt) { +typedef typename std::iterator_traits<string_type>::value_type char_type; + sarray_type RA; + index_type i, j, m, p, q, plen, qlen, name, pidx = 0; + bool diff; + int c; +#ifdef _OPENMP + int maxthreads = omp_get_max_threads(); +#else +# define maxthreads 1 +#endif + char_type c0, c1; + + /* stage 1: reduce the problem by at least 1/2 + sort all the S-substrings */ + if(fs < (maxthreads * k)) { + index_type *C, *B; + if((C = new index_type[maxthreads * k]) == 0) { return -2; } + B = (1 < maxthreads) ? C + k : C; + getCounts(T, C, n, k); getBuckets(C, B, k, true); /* find ends of buckets */ +#ifdef _OPENMP +#pragma omp parallel for default(shared) private(i) +#endif + for(i = 0; i < n; ++i) { SA[i] = 0; } + for(i = n - 2, c = 0, c1 = T[n - 1]; 0 <= i; --i, c1 = c0) { + if((c0 = T[i]) < (c1 + c)) { c = 1; } + else if(c != 0) { SA[--B[c1]] = i + 1, c = 0; } + } + induceSA(T, SA, C, B, n, k); + delete [] C; + } else { + sarray_type C, B; + C = SA + n; + B = ((1 < maxthreads) || (k <= (fs - k))) ? C + k : C; + getCounts(T, C, n, k); getBuckets(C, B, k, true); /* find ends of buckets */ +#ifdef _OPENMP +#pragma omp parallel for default(shared) private(i) +#endif + for(i = 0; i < n; ++i) { SA[i] = 0; } + for(i = n - 2, c = 0, c1 = T[n - 1]; 0 <= i; --i, c1 = c0) { + if((c0 = T[i]) < (c1 + c)) { c = 1; } + else if(c != 0) { SA[--B[c1]] = i + 1, c = 0; } + } + induceSA(T, SA, C, B, n, k); + } + + /* compact all the sorted substrings into the first m items of SA + 2*m must be not larger than n (proveable) */ +#ifdef _OPENMP +#pragma omp parallel for default(shared) private(i, j, p, c0, c1) + for(i = 0; i < n; ++i) { + p = SA[i]; + if((0 < p) && (T[p - 1] > (c0 = T[p]))) { + for(j = p + 1; (j < n) && (c0 == (c1 = T[j])); ++j) { } + if((j < n) && (c0 < c1)) { SA[i] = ~p; } + } + } + for(i = 0, m = 0; i < n; ++i) { if((p = SA[i]) < 0) { SA[m++] = ~p; } } +#else + for(i = 0, m = 0; i < n; ++i) { + p = SA[i]; + if((0 < p) && (T[p - 1] > (c0 = T[p]))) { + for(j = p + 1; (j < n) && (c0 == (c1 = T[j])); ++j) { } + if((j < n) && (c0 < c1)) { SA[m++] = p; } + } + } +#endif + j = m + (n >> 1); +#ifdef _OPENMP +#pragma omp parallel for default(shared) private(i) +#endif + for(i = m; i < j; ++i) { SA[i] = 0; } /* init the name array buffer */ + /* store the length of all substrings */ + for(i = n - 2, j = n, c = 0, c1 = T[n - 1]; 0 <= i; --i, c1 = c0) { + if((c0 = T[i]) < (c1 + c)) { c = 1; } + else if(c != 0) { SA[m + ((i + 1) >> 1)] = j - i - 1; j = i + 1; c = 0; } + } + /* find the lexicographic names of all substrings */ + for(i = 0, name = 0, q = n, qlen = 0; i < m; ++i) { + p = SA[i], plen = SA[m + (p >> 1)], diff = true; + if(plen == qlen) { + for(j = 0; (j < plen) && (T[p + j] == T[q + j]); ++j) { } + if(j == plen) { diff = false; } + } + if(diff != false) { ++name, q = p, qlen = plen; } + SA[m + (p >> 1)] = name; + } + + /* stage 2: solve the reduced problem + recurse if names are not yet unique */ + if(name < m) { + RA = SA + n + fs - m; + for(i = m + (n >> 1) - 1, j = m - 1; m <= i; --i) { + if(SA[i] != 0) { RA[j--] = SA[i] - 1; } + } + if(suffixsort(RA, SA, fs + n - m * 2, m, name, false) != 0) { return -2; } + for(i = n - 2, j = m - 1, c = 0, c1 = T[n - 1]; 0 <= i; --i, c1 = c0) { + if((c0 = T[i]) < (c1 + c)) { c = 1; } + else if(c != 0) { RA[j--] = i + 1, c = 0; } /* get p1 */ + } +#ifdef _OPENMP +#pragma omp parallel for default(shared) private(i) +#endif + for(i = 0; i < m; ++i) { SA[i] = RA[SA[i]]; } /* get index in s */ + } + + /* stage 3: induce the result for the original problem */ + if(fs < (maxthreads * k)) { + index_type *B, *C; + if((C = new index_type[maxthreads * k]) == 0) { return -2; } + B = (1 < maxthreads) ? C + k : C; + /* put all left-most S characters into their buckets */ + getCounts(T, C, n, k); getBuckets(C, B, k, true); /* find ends of buckets */ +#ifdef _OPENMP +#pragma omp parallel for default(shared) private(i) +#endif + for(i = m; i < n; ++i) { SA[i] = 0; } /* init SA[m..n-1] */ + for(i = m - 1; 0 <= i; --i) { + j = SA[i], SA[i] = 0; + SA[--B[T[j]]] = j; + } + if(isbwt == false) { induceSA(T, SA, C, B, n, k); } + else { pidx = computeBWT(T, SA, C, B, n, k); } + delete [] C; + } else { + sarray_type C, B; + C = SA + n; + B = ((1 < maxthreads) || (k <= (fs - k))) ? C + k : C; + /* put all left-most S characters into their buckets */ + getCounts(T, C, n, k); getBuckets(C, B, k, true); /* find ends of buckets */ +#ifdef _OPENMP +#pragma omp parallel for default(shared) private(i) +#endif + for(i = m; i < n; ++i) { SA[i] = 0; } /* init SA[m..n-1] */ + for(i = m - 1; 0 <= i; --i) { + j = SA[i], SA[i] = 0; + SA[--B[T[j]]] = j; + } + if(isbwt == false) { induceSA(T, SA, C, B, n, k); } + else { pidx = computeBWT(T, SA, C, B, n, k); } + } + + return pidx; +#ifndef _OPENMP +# undef maxthreads +#endif +} + +} /* namespace saisxx_private */ + + +/** + * @brief Constructs the suffix array of a given string in linear time. + * @param T[0..n-1] The input string. (random access iterator) + * @param SA[0..n-1] The output array of suffixes. (random access iterator) + * @param n The length of the given string. + * @param k The alphabet size. + * @return 0 if no error occurred, -1 or -2 otherwise. + */ +template<typename string_type, typename sarray_type, typename index_type> +int +saisxx(string_type T, sarray_type SA, index_type n, index_type k = 256) { + int err; + if((n < 0) || (k <= 0)) { return -1; } + if(n <= 1) { if(n == 1) { SA[0] = 0; } return 0; } + try { err = saisxx_private::suffixsort(T, SA, 0, n, k, false); } + catch(...) { err = -2; } + return err; +} + +/** + * @brief Constructs the burrows-wheeler transformed string of a given string in linear time. + * @param T[0..n-1] The input string. (random access iterator) + * @param U[0..n-1] The output string. (random access iterator) + * @param A[0..n-1] The temporary array. (random access iterator) + * @param n The length of the given string. + * @param k The alphabet size. + * @return The primary index if no error occurred, -1 or -2 otherwise. + */ +template<typename string_type, typename sarray_type, typename index_type> +index_type +saisxx_bwt(string_type T, string_type U, sarray_type A, index_type n, index_type k = 256) { +typedef typename std::iterator_traits<string_type>::value_type char_type; + index_type i, pidx; + if((n < 0) || (k <= 0)) { return -1; } + if(n <= 1) { if(n == 1) { U[0] = T[0]; } return n; } + try { + pidx = saisxx_private::suffixsort(T, A, 0, n, k, true); + if(0 <= pidx) { + U[0] = T[n - 1]; + for(i = 0; i < pidx; ++i) { U[i + 1] = (char_type)A[i]; } + for(i += 1; i < n; ++i) { U[i] = (char_type)A[i]; } + pidx += 1; + } + } catch(...) { pidx = -2; } + return pidx; +} + + +#endif /* __cplusplus */ +#endif /* _SAIS_HXX */ diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/esaxx/waf b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/esaxx/waf Binary files differnew file mode 100755 index 000000000..d99146fa4 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/esaxx/waf diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/esaxx/wscript b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/esaxx/wscript new file mode 100644 index 000000000..a58f45826 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/esaxx/wscript @@ -0,0 +1,23 @@ +VERSION= '0.0.4' +APPNAME= 'esaxx' + +srcdir= '.' +blddir= 'bin' + +def set_options(ctx): + ctx.tool_options('compiler_cxx') + +def configure(ctx): + ctx.check_tool('compiler_cxx') + ctx.env.CXXFLAGS += ['-O2', '-Wall', '-g'] + +def build(bld): + task1= bld(features='cxx cprogram', + source = 'enumSubstring.cpp', + name = 'enum_substring', + target = 'enum_substring', + includes = '.') + +def dist_hook(): + import os + os.remove('googlecode_upload.py') diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/find_opt_references.cc b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/find_opt_references.cc new file mode 100644 index 000000000..378d3b521 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/find_opt_references.cc @@ -0,0 +1,267 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + Author: zip753@gmail.com (Ivan Nikulin) + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* Tool for generating optimal backward references for the input file. Uses + sais-lite library for building suffix array. */ + +#include <algorithm> +#include <cassert> +#include <cstdio> +#include <cstring> +#include <functional> +#include <utility> +#include <vector> + +#include <gflags/gflags.h> +using gflags::ParseCommandLineFlags; + +#include "./esaxx/sais.hxx" + +DEFINE_bool(advanced, false, "Advanced searching mode: finds all longest " + "matches at positions that are not covered by matches of length at least " + "max_length. WARNING: uses much more memory than simple mode, especially " + "for small values of min_length."); +DEFINE_int32(min_length, 1, "Minimal length of found backward references."); +/* For advanced mode. */ +DEFINE_int32(long_length, 32, + "Maximal length of found backward references for advanced mode."); +DEFINE_int32(skip, 1, "Number of bytes to skip."); + +const size_t kFileBufferSize = (1 << 16); // 64KB + +typedef int sarray_type; // Can't make it unsigned because of templates :( +typedef uint8_t input_type; +typedef uint32_t lcp_type; +typedef std::pair<int, std::vector<int> > entry_type; +typedef std::function<void(sarray_type*, lcp_type*, size_t, int, int, int, int, + int)> Fn; + +void ReadInput(FILE* fin, input_type* storage, size_t input_size) { + size_t last_pos = 0; + size_t available_in; + fseek(fin, 0, SEEK_SET); + do { + available_in = fread(storage + last_pos, 1, kFileBufferSize, fin); + last_pos += available_in; + } while (available_in != 0); + assert(last_pos == input_size); +} + +void BuildLCP(input_type* storage, sarray_type* sarray, lcp_type* lcp, + size_t size, uint32_t* pos) { + for (int i = 0; i < size; ++i) { + pos[sarray[i]] = i; + } + uint32_t k = 0; + lcp[size - 1] = 0; + for (int i = 0; i < size; ++i) { + if (pos[i] == size - 1) { + k = 0; + continue; + } + uint32_t j = sarray[pos[i] + 1]; // Suffix which follow i-th suffix in SA. + while (i + k < size && j + k < size && storage[i + k] == storage[j + k]) { + ++k; + } + lcp[pos[i]] = k; + if (k > 0) --k; + } +} + +inline void PrintReference(sarray_type* sarray, lcp_type* lcp, size_t size, + int idx, int left_ix, int right_ix, int left_lcp, + int right_lcp, FILE* fout) { + int max_lcp_ix; + if (right_ix == size - 1 || (left_ix >= 0 && left_lcp >= right_lcp)) { + max_lcp_ix = left_ix; + } else { + max_lcp_ix = right_ix; + } + int dist = idx - sarray[max_lcp_ix]; + assert(dist > 0); + fputc(1, fout); + fwrite(&idx, sizeof(int), 1, fout); // Position in input. + fwrite(&dist, sizeof(int), 1, fout); // Backward distance. +} + +inline void GoLeft(sarray_type* sarray, lcp_type* lcp, int idx, int left_ix, + int left_lcp, entry_type* entry) { + entry->first = left_lcp; + if (left_lcp > FLAGS_long_length) return; + for (; left_ix >= 0; --left_ix) { + if (lcp[left_ix] < left_lcp) break; + if (sarray[left_ix] < idx) { + entry->second.push_back(idx - sarray[left_ix]); + } + } +} + +inline void GoRight(sarray_type* sarray, lcp_type* lcp, int idx, size_t size, + int right_ix, int right_lcp, entry_type* entry) { + entry->first = right_lcp; + if (right_lcp > FLAGS_long_length) return; + for (; right_ix < size - 1; ++right_ix) { + if (lcp[right_ix] < right_lcp) break; + if (sarray[right_ix] < idx) { + entry->second.push_back(idx - sarray[right_ix]); + } + } +} + +inline void StoreReference(sarray_type* sarray, lcp_type* lcp, size_t size, + int idx, int left_ix, int right_ix, int left_lcp, + int right_lcp, entry_type* entries) { + if (right_ix == size - 1 || (left_ix >= 0 && left_lcp > right_lcp)) { + // right is invalid or left is better + GoLeft(sarray, lcp, idx, left_ix, left_lcp, &entries[idx]); + } else if (left_ix < 0 || (right_ix < size - 1 && right_lcp > left_lcp)) { + // left is invalid or right is better + GoRight(sarray, lcp, idx, size, right_ix, right_lcp, &entries[idx]); + } else { // both are valid and of equal length + GoLeft(sarray, lcp, idx, left_ix, left_lcp, &entries[idx]); + GoRight(sarray, lcp, idx, size, right_ix, right_lcp, &entries[idx]); + } +} + +void ProcessReferences(sarray_type* sarray, lcp_type* lcp, size_t size, + uint32_t* pos, const Fn& process_output) { + int min_length = FLAGS_min_length; + for (int idx = FLAGS_skip; idx < size; ++idx) { + int left_lcp = -1; + int left_ix; + for (left_ix = pos[idx] - 1; left_ix >= 0; --left_ix) { + if (left_lcp == -1 || left_lcp > lcp[left_ix]) { + left_lcp = lcp[left_ix]; + } + if (left_lcp == 0) break; + if (sarray[left_ix] < idx) break; + } + + int right_lcp = -1; + int right_ix; + for (right_ix = pos[idx]; right_ix < size - 1; ++right_ix) { + if (right_lcp == -1 || right_lcp > lcp[right_ix]) { + right_lcp = lcp[right_ix]; + } + // Stop if we have better result from the left side already. + if (right_lcp < left_lcp && left_ix >= 0) break; + if (right_lcp == 0) break; + if (sarray[right_ix] < idx) break; + } + + if ((left_ix >= 0 && left_lcp >= min_length) || + (right_ix < size - 1 && right_lcp >= min_length)) { + process_output(sarray, lcp, size, idx, left_ix, right_ix, left_lcp, + right_lcp); + } + } +} + +void ProcessEntries(entry_type* entries, size_t size, FILE* fout) { + int long_length = FLAGS_long_length; + std::vector<std::pair<int, int> > segments; + size_t idx; + for (idx = 0; idx < size;) { + entry_type& entry = entries[idx]; + if (entry.first > long_length) { + // Add segment. + if (segments.empty() || segments.back().second < idx) { + segments.push_back({idx, idx + entry.first}); + } else { + segments.back().second = idx + entry.first; + } + } + ++idx; + } + printf("Segments generated.\n"); + size_t segments_ix = 0; + for (idx = 0; idx < size;) { + if (idx == segments[segments_ix].first) { + // Skip segment. + idx = segments[segments_ix].second; + } else { + for (auto& dist : entries[idx].second) { + fputc(1, fout); + fwrite(&idx, sizeof(int), 1, fout); // Position in input. + fwrite(&dist, sizeof(int), 1, fout); // Backward distance. + } + ++idx; + } + } +} + +int main(int argc, char* argv[]) { + ParseCommandLineFlags(&argc, &argv, true); + if (argc != 3) { + printf("usage: %s input_file output_file\n", argv[0]); + return 1; + } + + FILE* fin = fopen(argv[1], "rb"); + FILE* fout = fopen(argv[2], "w"); + + fseek(fin, 0, SEEK_END); + int input_size = ftell(fin); + fseek(fin, 0, SEEK_SET); + printf("The file size is %u bytes\n", input_size); + + input_type* storage = new input_type[input_size]; + + ReadInput(fin, storage, input_size); + fclose(fin); + + sarray_type* sarray = new sarray_type[input_size]; + saisxx(storage, sarray, input_size); + printf("Suffix array calculated.\n"); + + // Inverse suffix array. + uint32_t* pos = new uint32_t[input_size]; + + lcp_type* lcp = new lcp_type[input_size]; + BuildLCP(storage, sarray, lcp, input_size, pos); + printf("LCP array constructed.\n"); + delete[] storage; + + using std::placeholders::_1; + using std::placeholders::_2; + using std::placeholders::_3; + using std::placeholders::_4; + using std::placeholders::_5; + using std::placeholders::_6; + using std::placeholders::_7; + using std::placeholders::_8; + entry_type* entries; + if (FLAGS_advanced) { + entries = new entry_type[input_size]; + for (size_t i = 0; i < input_size; ++i) entries[i].first = -1; + } + Fn print = std::bind(PrintReference, _1, _2, _3, _4, _5, _6, _7, _8, fout); + Fn store = std::bind(StoreReference, _1, _2, _3, _4, _5, _6, _7, _8, entries); + + ProcessReferences(sarray, lcp, input_size, pos, + FLAGS_advanced ? store : print); + printf("References processed.\n"); + + if (FLAGS_advanced) { + int good_cnt = 0; + uint64_t avg_cnt = 0; + for (size_t i = 0; i < input_size; ++i) { + if (entries[i].first != -1) { + ++good_cnt; + avg_cnt += entries[i].second.size(); + } + } + printf("Number of covered positions = %d\n", good_cnt); + printf("Average number of references per covered position = %.4lf\n", + static_cast<double>(avg_cnt) / good_cnt); + ProcessEntries(entries, input_size, fout); + printf("Entries processed.\n"); + } + + fclose(fout); + return 0; +} diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/img/enwik9_brotli.png b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/img/enwik9_brotli.png Binary files differnew file mode 100644 index 000000000..c95eba967 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/img/enwik9_brotli.png diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/img/enwik9_diff.png b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/img/enwik9_diff.png Binary files differnew file mode 100644 index 000000000..5df6748e8 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/img/enwik9_diff.png diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/img/enwik9_opt.png b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/img/enwik9_opt.png Binary files differnew file mode 100644 index 000000000..43c655d44 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/img/enwik9_opt.png diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/.gitignore b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/.gitignore new file mode 100644 index 000000000..a0a066640 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/.gitignore @@ -0,0 +1,32 @@ +# Object files +*.o +*.ko +*.obj +*.elf + +# Precompiled Headers +*.gch +*.pch + +# Libraries +*.lib +*.a +*.la +*.lo + +# Shared objects (inc. Windows DLLs) +*.dll +*.so +*.so.* +*.dylib + +# Executables +*.exe +*.out +*.app +*.i*86 +*.x86_64 +*.hex + +# CMake files/directories +build/ diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/CHANGELOG.md b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/CHANGELOG.md new file mode 100644 index 000000000..fe9d00405 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/CHANGELOG.md @@ -0,0 +1,21 @@ +# libdivsufsort Change Log + +See full changelog at: https://github.com/y-256/libdivsufsort/commits + +## [2.0.1] - 2010-11-11 +### Fixed +* Wrong variable used in `divbwt` function +* Enclose some string variables with double quotation marks in include/CMakeLists.txt +* Fix typo in include/CMakeLists.txt + +## 2.0.0 - 2008-08-23 +### Changed +* Switch the build system to [CMake](http://www.cmake.org/) +* Improve the performance of the suffix-sorting algorithm + +### Added +* OpenMP support +* 64-bit version of divsufsort + +[Unreleased]: https://github.com/y-256/libdivsufsort/compare/2.0.1...HEAD +[2.0.1]: https://github.com/y-256/libdivsufsort/compare/2.0.0...2.0.1 diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/CMakeLists.txt b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/CMakeLists.txt new file mode 100644 index 000000000..785994393 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/CMakeLists.txt @@ -0,0 +1,99 @@ +### cmake file for building libdivsufsort Package ### +cmake_minimum_required(VERSION 2.4.4) +set(CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules") +include(AppendCompilerFlags) + +## Project information ## +project(libdivsufsort C) +set(PROJECT_VENDOR "Yuta Mori") +set(PROJECT_CONTACT "yuta.256@gmail.com") +set(PROJECT_URL "https://github.com/y-256/libdivsufsort") +set(PROJECT_DESCRIPTION "A lightweight suffix sorting library") +include(VERSION.cmake) + +## CPack configuration ## +set(CPACK_GENERATOR "TGZ;TBZ2;ZIP") +set(CPACK_SOURCE_GENERATOR "TGZ;TBZ2;ZIP") +include(ProjectCPack) + +## Project options ## +option(BUILD_SHARED_LIBS "Set to OFF to build static libraries" ON) +option(BUILD_EXAMPLES "Build examples" ON) +option(BUILD_DIVSUFSORT64 "Build libdivsufsort64" OFF) +option(USE_OPENMP "Use OpenMP for parallelization" OFF) +option(WITH_LFS "Enable Large File Support" ON) + +## Installation directories ## +set(LIB_SUFFIX "" CACHE STRING "Define suffix of directory name (32 or 64)") + +set(CMAKE_INSTALL_RUNTIMEDIR "" CACHE PATH "Specify the output directory for dll runtimes (default is bin)") +if(NOT CMAKE_INSTALL_RUNTIMEDIR) + set(CMAKE_INSTALL_RUNTIMEDIR "${CMAKE_INSTALL_PREFIX}/bin") +endif(NOT CMAKE_INSTALL_RUNTIMEDIR) + +set(CMAKE_INSTALL_LIBDIR "" CACHE PATH "Specify the output directory for libraries (default is lib)") +if(NOT CMAKE_INSTALL_LIBDIR) + set(CMAKE_INSTALL_LIBDIR "${CMAKE_INSTALL_PREFIX}/lib${LIB_SUFFIX}") +endif(NOT CMAKE_INSTALL_LIBDIR) + +set(CMAKE_INSTALL_INCLUDEDIR "" CACHE PATH "Specify the output directory for header files (default is include)") +if(NOT CMAKE_INSTALL_INCLUDEDIR) + set(CMAKE_INSTALL_INCLUDEDIR "${CMAKE_INSTALL_PREFIX}/include") +endif(NOT CMAKE_INSTALL_INCLUDEDIR) + +set(CMAKE_INSTALL_PKGCONFIGDIR "" CACHE PATH "Specify the output directory for pkgconfig files (default is lib/pkgconfig)") +if(NOT CMAKE_INSTALL_PKGCONFIGDIR) + set(CMAKE_INSTALL_PKGCONFIGDIR "${CMAKE_INSTALL_LIBDIR}/pkgconfig") +endif(NOT CMAKE_INSTALL_PKGCONFIGDIR) + +## Build type ## +if(NOT CMAKE_BUILD_TYPE) + set(CMAKE_BUILD_TYPE "Release") +elseif(CMAKE_BUILD_TYPE STREQUAL "Debug") + set(CMAKE_VERBOSE_MAKEFILE ON) +endif(NOT CMAKE_BUILD_TYPE) + +## Compiler options ## +if(MSVC) + append_c_compiler_flags("/W4" "VC" CMAKE_C_FLAGS) + append_c_compiler_flags("/Oi;/Ot;/Ox;/Oy" "VC" CMAKE_C_FLAGS_RELEASE) + if(USE_OPENMP) + append_c_compiler_flags("/openmp" "VC" CMAKE_C_FLAGS) + endif(USE_OPENMP) +elseif(BORLAND) + append_c_compiler_flags("-w" "BCC" CMAKE_C_FLAGS) + append_c_compiler_flags("-Oi;-Og;-Os;-Ov;-Ox" "BCC" CMAKE_C_FLAGS_RELEASE) +else(MSVC) + if(CMAKE_COMPILER_IS_GNUCC) + append_c_compiler_flags("-Wall" "GCC" CMAKE_C_FLAGS) + append_c_compiler_flags("-fomit-frame-pointer" "GCC" CMAKE_C_FLAGS_RELEASE) + if(USE_OPENMP) + append_c_compiler_flags("-fopenmp" "GCC" CMAKE_C_FLAGS) + endif(USE_OPENMP) + else(CMAKE_COMPILER_IS_GNUCC) + append_c_compiler_flags("-Wall" "UNKNOWN" CMAKE_C_FLAGS) + append_c_compiler_flags("-fomit-frame-pointer" "UNKNOWN" CMAKE_C_FLAGS_RELEASE) + if(USE_OPENMP) + append_c_compiler_flags("-fopenmp;-openmp;-omp" "UNKNOWN" CMAKE_C_FLAGS) + endif(USE_OPENMP) + endif(CMAKE_COMPILER_IS_GNUCC) +endif(MSVC) + +## Add definitions ## +add_definitions(-DHAVE_CONFIG_H=1 -D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS) + +## Add subdirectories ## +add_subdirectory(pkgconfig) +add_subdirectory(include) +add_subdirectory(lib) +if(BUILD_EXAMPLES) + add_subdirectory(examples) +endif(BUILD_EXAMPLES) + +## Add 'uninstall' target ## +CONFIGURE_FILE( + "${CMAKE_CURRENT_SOURCE_DIR}/CMakeModules/cmake_uninstall.cmake.in" + "${CMAKE_CURRENT_BINARY_DIR}/CMakeModules/cmake_uninstall.cmake" + IMMEDIATE @ONLY) +ADD_CUSTOM_TARGET(uninstall + "${CMAKE_COMMAND}" -P "${CMAKE_CURRENT_BINARY_DIR}/CMakeModules/cmake_uninstall.cmake") diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/CMakeModules/AppendCompilerFlags.cmake b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/CMakeModules/AppendCompilerFlags.cmake new file mode 100644 index 000000000..58d3f99ec --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/CMakeModules/AppendCompilerFlags.cmake @@ -0,0 +1,38 @@ +include(CheckCSourceCompiles) +include(CheckCXXSourceCompiles) + +macro(append_c_compiler_flags _flags _name _result) + set(SAFE_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS}) + string(REGEX REPLACE "[-+/ ]" "_" cname "${_name}") + string(TOUPPER "${cname}" cname) + foreach(flag ${_flags}) + string(REGEX REPLACE "^[-+/ ]+(.*)[-+/ ]*$" "\\1" flagname "${flag}") + string(REGEX REPLACE "[-+/ ]" "_" flagname "${flagname}") + string(TOUPPER "${flagname}" flagname) + set(have_flag "HAVE_${cname}_${flagname}") + set(CMAKE_REQUIRED_FLAGS "${flag}") + check_c_source_compiles("int main() { return 0; }" ${have_flag}) + if(${have_flag}) + set(${_result} "${${_result}} ${flag}") + endif(${have_flag}) + endforeach(flag) + set(CMAKE_REQUIRED_FLAGS ${SAFE_CMAKE_REQUIRED_FLAGS}) +endmacro(append_c_compiler_flags) + +macro(append_cxx_compiler_flags _flags _name _result) + set(SAFE_CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS}) + string(REGEX REPLACE "[-+/ ]" "_" cname "${_name}") + string(TOUPPER "${cname}" cname) + foreach(flag ${_flags}) + string(REGEX REPLACE "^[-+/ ]+(.*)[-+/ ]*$" "\\1" flagname "${flag}") + string(REGEX REPLACE "[-+/ ]" "_" flagname "${flagname}") + string(TOUPPER "${flagname}" flagname) + set(have_flag "HAVE_${cname}_${flagname}") + set(CMAKE_REQUIRED_FLAGS "${flag}") + check_cxx_source_compiles("int main() { return 0; }" ${have_flag}) + if(${have_flag}) + set(${_result} "${${_result}} ${flag}") + endif(${have_flag}) + endforeach(flag) + set(CMAKE_REQUIRED_FLAGS ${SAFE_CMAKE_REQUIRED_FLAGS}) +endmacro(append_cxx_compiler_flags) diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/CMakeModules/CheckFunctionKeywords.cmake b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/CMakeModules/CheckFunctionKeywords.cmake new file mode 100644 index 000000000..44601fd4e --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/CMakeModules/CheckFunctionKeywords.cmake @@ -0,0 +1,15 @@ +include(CheckCSourceCompiles) + +macro(check_function_keywords _wordlist) + set(${_result} "") + foreach(flag ${_wordlist}) + string(REGEX REPLACE "[-+/ ()]" "_" flagname "${flag}") + string(TOUPPER "${flagname}" flagname) + set(have_flag "HAVE_${flagname}") + check_c_source_compiles("${flag} void func(); void func() { } int main() { func(); return 0; }" ${have_flag}) + if(${have_flag} AND NOT ${_result}) + set(${_result} "${flag}") +# break() + endif(${have_flag} AND NOT ${_result}) + endforeach(flag) +endmacro(check_function_keywords) diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/CMakeModules/CheckLFS.cmake b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/CMakeModules/CheckLFS.cmake new file mode 100644 index 000000000..e2b009914 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/CMakeModules/CheckLFS.cmake @@ -0,0 +1,109 @@ +## Checks for large file support ## +include(CheckIncludeFile) +include(CheckSymbolExists) +include(CheckTypeSize) + +macro(check_lfs _isenable) + set(LFS_OFF_T "") + set(LFS_FOPEN "") + set(LFS_FSEEK "") + set(LFS_FTELL "") + set(LFS_PRID "") + + if(${_isenable}) + set(SAFE_CMAKE_REQUIRED_DEFINITIONS "${CMAKE_REQUIRED_DEFINITIONS}") + set(CMAKE_REQUIRED_DEFINITIONS ${CMAKE_REQUIRED_DEFINITIONS} + -D_LARGEFILE_SOURCE -D_LARGE_FILES -D_FILE_OFFSET_BITS=64 + -D__STDC_LIMIT_MACROS -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS) + + check_include_file("sys/types.h" HAVE_SYS_TYPES_H) + check_include_file("inttypes.h" HAVE_INTTYPES_H) + check_include_file("stddef.h" HAVE_STDDEF_H) + check_include_file("stdint.h" HAVE_STDINT_H) + + # LFS type1: 8 <= sizeof(off_t), fseeko, ftello + check_type_size("off_t" SIZEOF_OFF_T) + if(SIZEOF_OFF_T GREATER 7) + check_symbol_exists("fseeko" "stdio.h" HAVE_FSEEKO) + check_symbol_exists("ftello" "stdio.h" HAVE_FTELLO) + if(HAVE_FSEEKO AND HAVE_FTELLO) + set(LFS_OFF_T "off_t") + set(LFS_FOPEN "fopen") + set(LFS_FSEEK "fseeko") + set(LFS_FTELL "ftello") + check_symbol_exists("PRIdMAX" "inttypes.h" HAVE_PRIDMAX) + if(HAVE_PRIDMAX) + set(LFS_PRID "PRIdMAX") + else(HAVE_PRIDMAX) + check_type_size("long" SIZEOF_LONG) + check_type_size("int" SIZEOF_INT) + if(SIZEOF_OFF_T GREATER SIZEOF_LONG) + set(LFS_PRID "\"lld\"") + elseif(SIZEOF_LONG GREATER SIZEOF_INT) + set(LFS_PRID "\"ld\"") + else(SIZEOF_OFF_T GREATER SIZEOF_LONG) + set(LFS_PRID "\"d\"") + endif(SIZEOF_OFF_T GREATER SIZEOF_LONG) + endif(HAVE_PRIDMAX) + endif(HAVE_FSEEKO AND HAVE_FTELLO) + endif(SIZEOF_OFF_T GREATER 7) + + # LFS type2: 8 <= sizeof(off64_t), fopen64, fseeko64, ftello64 + if(NOT LFS_OFF_T) + check_type_size("off64_t" SIZEOF_OFF64_T) + if(SIZEOF_OFF64_T GREATER 7) + check_symbol_exists("fopen64" "stdio.h" HAVE_FOPEN64) + check_symbol_exists("fseeko64" "stdio.h" HAVE_FSEEKO64) + check_symbol_exists("ftello64" "stdio.h" HAVE_FTELLO64) + if(HAVE_FOPEN64 AND HAVE_FSEEKO64 AND HAVE_FTELLO64) + set(LFS_OFF_T "off64_t") + set(LFS_FOPEN "fopen64") + set(LFS_FSEEK "fseeko64") + set(LFS_FTELL "ftello64") + check_symbol_exists("PRIdMAX" "inttypes.h" HAVE_PRIDMAX) + if(HAVE_PRIDMAX) + set(LFS_PRID "PRIdMAX") + else(HAVE_PRIDMAX) + check_type_size("long" SIZEOF_LONG) + check_type_size("int" SIZEOF_INT) + if(SIZEOF_OFF64_T GREATER SIZEOF_LONG) + set(LFS_PRID "\"lld\"") + elseif(SIZEOF_LONG GREATER SIZEOF_INT) + set(LFS_PRID "\"ld\"") + else(SIZEOF_OFF64_T GREATER SIZEOF_LONG) + set(LFS_PRID "\"d\"") + endif(SIZEOF_OFF64_T GREATER SIZEOF_LONG) + endif(HAVE_PRIDMAX) + endif(HAVE_FOPEN64 AND HAVE_FSEEKO64 AND HAVE_FTELLO64) + endif(SIZEOF_OFF64_T GREATER 7) + endif(NOT LFS_OFF_T) + + # LFS type3: 8 <= sizeof(__int64), _fseeki64, _ftelli64 + if(NOT LFS_OFF_T) + check_type_size("__int64" SIZEOF___INT64) + if(SIZEOF___INT64 GREATER 7) + check_symbol_exists("_fseeki64" "stdio.h" HAVE__FSEEKI64) + check_symbol_exists("_ftelli64" "stdio.h" HAVE__FTELLI64) + if(HAVE__FSEEKI64 AND HAVE__FTELLI64) + set(LFS_OFF_T "__int64") + set(LFS_FOPEN "fopen") + set(LFS_FSEEK "_fseeki64") + set(LFS_FTELL "_ftelli64") + set(LFS_PRID "\"I64d\"") + endif(HAVE__FSEEKI64 AND HAVE__FTELLI64) + endif(SIZEOF___INT64 GREATER 7) + endif(NOT LFS_OFF_T) + + set(CMAKE_REQUIRED_DEFINITIONS "${SAFE_CMAKE_REQUIRED_DEFINITIONS}") + endif(${_isenable}) + + if(NOT LFS_OFF_T) + ## not found + set(LFS_OFF_T "long") + set(LFS_FOPEN "fopen") + set(LFS_FSEEK "fseek") + set(LFS_FTELL "ftell") + set(LFS_PRID "\"ld\"") + endif(NOT LFS_OFF_T) + +endmacro(check_lfs) diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/CMakeModules/ProjectCPack.cmake b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/CMakeModules/ProjectCPack.cmake new file mode 100644 index 000000000..7c105f933 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/CMakeModules/ProjectCPack.cmake @@ -0,0 +1,38 @@ +# If the cmake version includes cpack, use it +IF(EXISTS "${CMAKE_ROOT}/Modules/CPack.cmake") + SET(CPACK_PACKAGE_DESCRIPTION_SUMMARY "${PROJECT_DESCRIPTION}") + SET(CPACK_PACKAGE_VENDOR "${PROJECT_VENDOR}") + SET(CPACK_PACKAGE_DESCRIPTION_FILE "${CMAKE_CURRENT_SOURCE_DIR}/README.md") + SET(CPACK_RESOURCE_FILE_LICENSE "${CMAKE_CURRENT_SOURCE_DIR}/LICENSE") + SET(CPACK_PACKAGE_VERSION_MAJOR "${PROJECT_VERSION_MAJOR}") + SET(CPACK_PACKAGE_VERSION_MINOR "${PROJECT_VERSION_MINOR}") + SET(CPACK_PACKAGE_VERSION_PATCH "${PROJECT_VERSION_PATCH}") +# SET(CPACK_PACKAGE_INSTALL_DIRECTORY "${PROJECT_NAME} ${PROJECT_VERSION}") + SET(CPACK_SOURCE_PACKAGE_FILE_NAME "${PROJECT_NAME}-${PROJECT_VERSION_FULL}") + + IF(NOT DEFINED CPACK_SYSTEM_NAME) + SET(CPACK_SYSTEM_NAME "${CMAKE_SYSTEM_NAME}-${CMAKE_SYSTEM_PROCESSOR}") + ENDIF(NOT DEFINED CPACK_SYSTEM_NAME) + + IF(${CPACK_SYSTEM_NAME} MATCHES Windows) + IF(CMAKE_CL_64) + SET(CPACK_SYSTEM_NAME win64-${CMAKE_SYSTEM_PROCESSOR}) + ELSE(CMAKE_CL_64) + SET(CPACK_SYSTEM_NAME win32-${CMAKE_SYSTEM_PROCESSOR}) + ENDIF(CMAKE_CL_64) + ENDIF(${CPACK_SYSTEM_NAME} MATCHES Windows) + + IF(NOT DEFINED CPACK_PACKAGE_FILE_NAME) + SET(CPACK_PACKAGE_FILE_NAME "${CPACK_SOURCE_PACKAGE_FILE_NAME}-${CPACK_SYSTEM_NAME}") + ENDIF(NOT DEFINED CPACK_PACKAGE_FILE_NAME) + + SET(CPACK_PACKAGE_CONTACT "${PROJECT_CONTACT}") + IF(UNIX) + SET(CPACK_STRIP_FILES "") + SET(CPACK_SOURCE_STRIP_FILES "") +# SET(CPACK_PACKAGE_EXECUTABLES "ccmake" "CMake") + ENDIF(UNIX) + SET(CPACK_SOURCE_IGNORE_FILES "/CVS/" "/build/" "/\\\\.build/" "/\\\\.svn/" "~$") + # include CPack model once all variables are set + INCLUDE(CPack) +ENDIF(EXISTS "${CMAKE_ROOT}/Modules/CPack.cmake") diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/CMakeModules/cmake_uninstall.cmake.in b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/CMakeModules/cmake_uninstall.cmake.in new file mode 100644 index 000000000..8366a8353 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/CMakeModules/cmake_uninstall.cmake.in @@ -0,0 +1,36 @@ +IF(NOT EXISTS "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt") + MESSAGE(FATAL_ERROR "Cannot find install manifest: \"@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt\"") +ENDIF(NOT EXISTS "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt") + +FILE(READ "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt" files) +STRING(REGEX REPLACE "\n" ";" files "${files}") + +SET(NUM 0) +FOREACH(file ${files}) + IF(EXISTS "$ENV{DESTDIR}${file}") + MESSAGE(STATUS "Looking for \"$ENV{DESTDIR}${file}\" - found") + SET(UNINSTALL_CHECK_${NUM} 1) + ELSE(EXISTS "$ENV{DESTDIR}${file}") + MESSAGE(STATUS "Looking for \"$ENV{DESTDIR}${file}\" - not found") + SET(UNINSTALL_CHECK_${NUM} 0) + ENDIF(EXISTS "$ENV{DESTDIR}${file}") + MATH(EXPR NUM "1 + ${NUM}") +ENDFOREACH(file) + +SET(NUM 0) +FOREACH(file ${files}) + IF(${UNINSTALL_CHECK_${NUM}}) + MESSAGE(STATUS "Uninstalling \"$ENV{DESTDIR}${file}\"") + EXEC_PROGRAM( + "@CMAKE_COMMAND@" ARGS "-E remove \"$ENV{DESTDIR}${file}\"" + OUTPUT_VARIABLE rm_out + RETURN_VALUE rm_retval + ) + IF(NOT "${rm_retval}" STREQUAL 0) + MESSAGE(FATAL_ERROR "Problem when removing \"$ENV{DESTDIR}${file}\"") + ENDIF(NOT "${rm_retval}" STREQUAL 0) + ENDIF(${UNINSTALL_CHECK_${NUM}}) + MATH(EXPR NUM "1 + ${NUM}") +ENDFOREACH(file) + +FILE(REMOVE "@CMAKE_CURRENT_BINARY_DIR@/install_manifest.txt") diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/LICENSE b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/LICENSE new file mode 100644 index 000000000..249efa49a --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2003 Yuta Mori All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/README.md b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/README.md new file mode 100644 index 000000000..381a18888 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/README.md @@ -0,0 +1,140 @@ +# libdivsufsort + +libdivsufsort is a software library that implements a lightweight suffix array construction algorithm. + +## News +* 2015-03-21: The project has moved from [Google Code](http://code.google.com/p/libdivsufsort/) to [GitHub](https://github.com/y-256/libdivsufsort) + +## Introduction +This library provides a simple and an efficient C API to construct a suffix array and a Burrows-Wheeler transformed string from a given string over a constant-size alphabet. +The algorithm runs in O(n log n) worst-case time using only 5n+O(1) bytes of memory space, where n is the length of +the string. + +## Build requirements +* An ANSI C Compiler (e.g. GNU GCC) +* [CMake](http://www.cmake.org/ "CMake") version 2.4.2 or newer +* CMake-supported build tool + +## Building on GNU/Linux +1. Get the source code from GitHub. You can either + * use git to clone the repository + ``` + git clone https://github.com/y-256/libdivsufsort.git + ``` + * or download a [zip file](../../archive/master.zip) directly +2. Create a `build` directory in the package source directory. +```shell +$ cd libdivsufsort +$ mkdir build +$ cd build +``` +3. Configure the package for your system. +If you want to install to a different location, change the -DCMAKE_INSTALL_PREFIX option. +```shell +$ cmake -DCMAKE_BUILD_TYPE="Release" \ +-DCMAKE_INSTALL_PREFIX="/usr/local" .. +``` +4. Compile the package. +```shell +$ make +``` +5. (Optional) Install the library and header files. +```shell +$ sudo make install +``` + +## API +```c +/* Data types */ +typedef int32_t saint_t; +typedef int32_t saidx_t; +typedef uint8_t sauchar_t; + +/* + * Constructs the suffix array of a given string. + * @param T[0..n-1] The input string. + * @param SA[0..n-1] The output array or suffixes. + * @param n The length of the given string. + * @return 0 if no error occurred, -1 or -2 otherwise. + */ +saint_t +divsufsort(const sauchar_t *T, saidx_t *SA, saidx_t n); + +/* + * Constructs the burrows-wheeler transformed string of a given string. + * @param T[0..n-1] The input string. + * @param U[0..n-1] The output string. (can be T) + * @param A[0..n-1] The temporary array. (can be NULL) + * @param n The length of the given string. + * @return The primary index if no error occurred, -1 or -2 otherwise. + */ +saidx_t +divbwt(const sauchar_t *T, sauchar_t *U, saidx_t *A, saidx_t n); +``` + +## Example Usage +```c +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include <divsufsort.h> + +int main() { + // intput data + char *Text = "abracadabra"; + int n = strlen(Text); + int i, j; + + // allocate + int *SA = (int *)malloc(n * sizeof(int)); + + // sort + divsufsort((unsigned char *)Text, SA, n); + + // output + for(i = 0; i < n; ++i) { + printf("SA[%2d] = %2d: ", i, SA[i]); + for(j = SA[i]; j < n; ++j) { + printf("%c", Text[j]); + } + printf("$\n"); + } + + // deallocate + free(SA); + + return 0; +} +``` +See the [examples](examples) directory for a few other examples. + +## Benchmarks +See [Benchmarks](https://github.com/y-256/libdivsufsort/blob/wiki/SACA_Benchmarks.md) page for details. + +## License +libdivsufsort is released under the [MIT license](LICENSE "MIT license"). +> The MIT License (MIT) +> +> Copyright (c) 2003 Yuta Mori All rights reserved. +> +> Permission is hereby granted, free of charge, to any person obtaining a copy +> of this software and associated documentation files (the "Software"), to deal +> in the Software without restriction, including without limitation the rights +> to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +> copies of the Software, and to permit persons to whom the Software is +> furnished to do so, subject to the following conditions: +> +> The above copyright notice and this permission notice shall be included in all +> copies or substantial portions of the Software. +> +> THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +> IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +> FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +> AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +> LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +> OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +> SOFTWARE. + +## Author +* Yuta Mori diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/VERSION.cmake b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/VERSION.cmake new file mode 100644 index 000000000..3f11ac18c --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/VERSION.cmake @@ -0,0 +1,23 @@ +set(PROJECT_VERSION_MAJOR "2") +set(PROJECT_VERSION_MINOR "0") +set(PROJECT_VERSION_PATCH "2") +set(PROJECT_VERSION_EXTRA "-1") +set(PROJECT_VERSION "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}") +set(PROJECT_VERSION_FULL "${PROJECT_VERSION_MAJOR}.${PROJECT_VERSION_MINOR}.${PROJECT_VERSION_PATCH}${PROJECT_VERSION_EXTRA}") + +set(LIBRARY_VERSION "3.0.1") +set(LIBRARY_SOVERSION "3") + +## Git revision number ## +if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git") + execute_process(COMMAND git describe --tags HEAD + WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" + OUTPUT_VARIABLE GIT_DESCRIBE_TAGS ERROR_QUIET) + if(GIT_DESCRIBE_TAGS) + string(REGEX REPLACE "^v(.*)" "\\1" GIT_REVISION "${GIT_DESCRIBE_TAGS}") + string(STRIP "${GIT_REVISION}" GIT_REVISION) + if(GIT_REVISION) + set(PROJECT_VERSION_FULL "${GIT_REVISION}") + endif(GIT_REVISION) + endif(GIT_DESCRIBE_TAGS) +endif(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git") diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/examples/CMakeLists.txt b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/examples/CMakeLists.txt new file mode 100644 index 000000000..e801c81a4 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/examples/CMakeLists.txt @@ -0,0 +1,11 @@ +## Add definitions ## +add_definitions(-D_LARGEFILE_SOURCE -D_LARGE_FILES -D_FILE_OFFSET_BITS=64) + +## Targets ## +include_directories("${CMAKE_CURRENT_SOURCE_DIR}/../include" + "${CMAKE_CURRENT_BINARY_DIR}/../include") +link_directories("${CMAKE_CURRENT_BINARY_DIR}/../lib") +foreach(src suftest mksary sasearch bwt unbwt) + add_executable(${src} ${src}.c) + target_link_libraries(${src} divsufsort) +endforeach(src) diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/examples/bwt.c b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/examples/bwt.c new file mode 100644 index 000000000..5a362d017 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/examples/bwt.c @@ -0,0 +1,220 @@ +/* + * bwt.c for libdivsufsort + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#if HAVE_CONFIG_H +# include "config.h" +#endif +#include <stdio.h> +#if HAVE_STRING_H +# include <string.h> +#endif +#if HAVE_STDLIB_H +# include <stdlib.h> +#endif +#if HAVE_MEMORY_H +# include <memory.h> +#endif +#if HAVE_STDDEF_H +# include <stddef.h> +#endif +#if HAVE_STRINGS_H +# include <strings.h> +#endif +#if HAVE_SYS_TYPES_H +# include <sys/types.h> +#endif +#if HAVE_IO_H && HAVE_FCNTL_H +# include <io.h> +# include <fcntl.h> +#endif +#include <time.h> +#include <divsufsort.h> +#include "lfs.h" + + +static +size_t +write_int(FILE *fp, saidx_t n) { + unsigned char c[4]; + c[0] = (unsigned char)((n >> 0) & 0xff), c[1] = (unsigned char)((n >> 8) & 0xff), + c[2] = (unsigned char)((n >> 16) & 0xff), c[3] = (unsigned char)((n >> 24) & 0xff); + return fwrite(c, sizeof(unsigned char), 4, fp); +} + +static +void +print_help(const char *progname, int status) { + fprintf(stderr, + "bwt, a burrows-wheeler transform program, version %s.\n", + divsufsort_version()); + fprintf(stderr, "usage: %s [-b num] INFILE OUTFILE\n", progname); + fprintf(stderr, " -b num set block size to num MiB [1..512] (default: 32)\n\n"); + exit(status); +} + +int +main(int argc, const char *argv[]) { + FILE *fp, *ofp; + const char *fname, *ofname; + sauchar_t *T; + saidx_t *SA; + LFS_OFF_T n; + size_t m; + saidx_t pidx; + clock_t start,finish; + saint_t i, blocksize = 32, needclose = 3; + + /* Check arguments. */ + if((argc == 1) || + (strcmp(argv[1], "-h") == 0) || + (strcmp(argv[1], "--help") == 0)) { print_help(argv[0], EXIT_SUCCESS); } + if((argc != 3) && (argc != 5)) { print_help(argv[0], EXIT_FAILURE); } + i = 1; + if(argc == 5) { + if(strcmp(argv[i], "-b") != 0) { print_help(argv[0], EXIT_FAILURE); } + blocksize = atoi(argv[i + 1]); + if(blocksize < 0) { blocksize = 1; } + else if(512 < blocksize) { blocksize = 512; } + i += 2; + } + blocksize <<= 20; + + /* Open a file for reading. */ + if(strcmp(argv[i], "-") != 0) { +#if HAVE_FOPEN_S + if(fopen_s(&fp, fname = argv[i], "rb") != 0) { +#else + if((fp = LFS_FOPEN(fname = argv[i], "rb")) == NULL) { +#endif + fprintf(stderr, "%s: Cannot open file `%s': ", argv[0], fname); + perror(NULL); + exit(EXIT_FAILURE); + } + } else { +#if HAVE__SETMODE && HAVE__FILENO + if(_setmode(_fileno(stdin), _O_BINARY) == -1) { + fprintf(stderr, "%s: Cannot set mode: ", argv[0]); + perror(NULL); + exit(EXIT_FAILURE); + } +#endif + fp = stdin; + fname = "stdin"; + needclose ^= 1; + } + i += 1; + + /* Open a file for writing. */ + if(strcmp(argv[i], "-") != 0) { +#if HAVE_FOPEN_S + if(fopen_s(&ofp, ofname = argv[i], "wb") != 0) { +#else + if((ofp = LFS_FOPEN(ofname = argv[i], "wb")) == NULL) { +#endif + fprintf(stderr, "%s: Cannot open file `%s': ", argv[0], ofname); + perror(NULL); + exit(EXIT_FAILURE); + } + } else { +#if HAVE__SETMODE && HAVE__FILENO + if(_setmode(_fileno(stdout), _O_BINARY) == -1) { + fprintf(stderr, "%s: Cannot set mode: ", argv[0]); + perror(NULL); + exit(EXIT_FAILURE); + } +#endif + ofp = stdout; + ofname = "stdout"; + needclose ^= 2; + } + + /* Get the file size. */ + if(LFS_FSEEK(fp, 0, SEEK_END) == 0) { + n = LFS_FTELL(fp); + rewind(fp); + if(n < 0) { + fprintf(stderr, "%s: Cannot ftell `%s': ", argv[0], fname); + perror(NULL); + exit(EXIT_FAILURE); + } + if(0x20000000L < n) { n = 0x20000000L; } + if((blocksize == 0) || (n < blocksize)) { blocksize = (saidx_t)n; } + } else if(blocksize == 0) { blocksize = 32 << 20; } + + /* Allocate 5blocksize bytes of memory. */ + T = (sauchar_t *)malloc(blocksize * sizeof(sauchar_t)); + SA = (saidx_t *)malloc(blocksize * sizeof(saidx_t)); + if((T == NULL) || (SA == NULL)) { + fprintf(stderr, "%s: Cannot allocate memory.\n", argv[0]); + exit(EXIT_FAILURE); + } + + /* Write the blocksize. */ + if(write_int(ofp, blocksize) != 4) { + fprintf(stderr, "%s: Cannot write to `%s': ", argv[0], ofname); + perror(NULL); + exit(EXIT_FAILURE); + } + + fprintf(stderr, " BWT (blocksize %" PRIdSAINT_T ") ... ", blocksize); + start = clock(); + for(n = 0; 0 < (m = fread(T, sizeof(sauchar_t), blocksize, fp)); n += m) { + /* Burrows-Wheeler Transform. */ + pidx = divbwt(T, T, SA, m); + if(pidx < 0) { + fprintf(stderr, "%s (bw_transform): %s.\n", + argv[0], + (pidx == -1) ? "Invalid arguments" : "Cannot allocate memory"); + exit(EXIT_FAILURE); + } + + /* Write the bwted data. */ + if((write_int(ofp, pidx) != 4) || + (fwrite(T, sizeof(sauchar_t), m, ofp) != m)) { + fprintf(stderr, "%s: Cannot write to `%s': ", argv[0], ofname); + perror(NULL); + exit(EXIT_FAILURE); + } + } + if(ferror(fp)) { + fprintf(stderr, "%s: Cannot read from `%s': ", argv[0], fname); + perror(NULL); + exit(EXIT_FAILURE); + } + finish = clock(); + fprintf(stderr, "%" PRIdOFF_T " bytes: %.4f sec\n", + n, (double)(finish - start) / (double)CLOCKS_PER_SEC); + + /* Close files */ + if(needclose & 1) { fclose(fp); } + if(needclose & 2) { fclose(ofp); } + + /* Deallocate memory. */ + free(SA); + free(T); + + return 0; +} diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/examples/mksary.c b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/examples/mksary.c new file mode 100644 index 000000000..b48177cf8 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/examples/mksary.c @@ -0,0 +1,193 @@ +/* + * mksary.c for libdivsufsort + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#if HAVE_CONFIG_H +# include "config.h" +#endif +#include <stdio.h> +#if HAVE_STRING_H +# include <string.h> +#endif +#if HAVE_STDLIB_H +# include <stdlib.h> +#endif +#if HAVE_MEMORY_H +# include <memory.h> +#endif +#if HAVE_STDDEF_H +# include <stddef.h> +#endif +#if HAVE_STRINGS_H +# include <strings.h> +#endif +#if HAVE_SYS_TYPES_H +# include <sys/types.h> +#endif +#if HAVE_IO_H && HAVE_FCNTL_H +# include <io.h> +# include <fcntl.h> +#endif +#include <time.h> +#include <divsufsort.h> +#include "lfs.h" + + +static +void +print_help(const char *progname, int status) { + fprintf(stderr, + "mksary, a simple suffix array builder, version %s.\n", + divsufsort_version()); + fprintf(stderr, "usage: %s INFILE OUTFILE\n\n", progname); + exit(status); +} + +int +main(int argc, const char *argv[]) { + FILE *fp, *ofp; + const char *fname, *ofname; + sauchar_t *T; + saidx_t *SA; + LFS_OFF_T n; + clock_t start, finish; + saint_t needclose = 3; + + /* Check arguments. */ + if((argc == 1) || + (strcmp(argv[1], "-h") == 0) || + (strcmp(argv[1], "--help") == 0)) { print_help(argv[0], EXIT_SUCCESS); } + if(argc != 3) { print_help(argv[0], EXIT_FAILURE); } + + /* Open a file for reading. */ + if(strcmp(argv[1], "-") != 0) { +#if HAVE_FOPEN_S + if(fopen_s(&fp, fname = argv[1], "rb") != 0) { +#else + if((fp = LFS_FOPEN(fname = argv[1], "rb")) == NULL) { +#endif + fprintf(stderr, "%s: Cannot open file `%s': ", argv[0], fname); + perror(NULL); + exit(EXIT_FAILURE); + } + } else { +#if HAVE__SETMODE && HAVE__FILENO + if(_setmode(_fileno(stdin), _O_BINARY) == -1) { + fprintf(stderr, "%s: Cannot set mode: ", argv[0]); + perror(NULL); + exit(EXIT_FAILURE); + } +#endif + fp = stdin; + fname = "stdin"; + needclose ^= 1; + } + + /* Open a file for writing. */ + if(strcmp(argv[2], "-") != 0) { +#if HAVE_FOPEN_S + if(fopen_s(&ofp, ofname = argv[2], "wb") != 0) { +#else + if((ofp = LFS_FOPEN(ofname = argv[2], "wb")) == NULL) { +#endif + fprintf(stderr, "%s: Cannot open file `%s': ", argv[0], ofname); + perror(NULL); + exit(EXIT_FAILURE); + } + } else { +#if HAVE__SETMODE && HAVE__FILENO + if(_setmode(_fileno(stdout), _O_BINARY) == -1) { + fprintf(stderr, "%s: Cannot set mode: ", argv[0]); + perror(NULL); + exit(EXIT_FAILURE); + } +#endif + ofp = stdout; + ofname = "stdout"; + needclose ^= 2; + } + + /* Get the file size. */ + if(LFS_FSEEK(fp, 0, SEEK_END) == 0) { + n = LFS_FTELL(fp); + rewind(fp); + if(n < 0) { + fprintf(stderr, "%s: Cannot ftell `%s': ", argv[0], fname); + perror(NULL); + exit(EXIT_FAILURE); + } + if(0x7fffffff <= n) { + fprintf(stderr, "%s: Input file `%s' is too big.\n", argv[0], fname); + exit(EXIT_FAILURE); + } + } else { + fprintf(stderr, "%s: Cannot fseek `%s': ", argv[0], fname); + perror(NULL); + exit(EXIT_FAILURE); + } + + /* Allocate 5blocksize bytes of memory. */ + T = (sauchar_t *)malloc((size_t)n * sizeof(sauchar_t)); + SA = (saidx_t *)malloc((size_t)n * sizeof(saidx_t)); + if((T == NULL) || (SA == NULL)) { + fprintf(stderr, "%s: Cannot allocate memory.\n", argv[0]); + exit(EXIT_FAILURE); + } + + /* Read n bytes of data. */ + if(fread(T, sizeof(sauchar_t), (size_t)n, fp) != (size_t)n) { + fprintf(stderr, "%s: %s `%s': ", + argv[0], + (ferror(fp) || !feof(fp)) ? "Cannot read from" : "Unexpected EOF in", + fname); + perror(NULL); + exit(EXIT_FAILURE); + } + if(needclose & 1) { fclose(fp); } + + /* Construct the suffix array. */ + fprintf(stderr, "%s: %" PRIdOFF_T " bytes ... ", fname, n); + start = clock(); + if(divsufsort(T, SA, (saidx_t)n) != 0) { + fprintf(stderr, "%s: Cannot allocate memory.\n", argv[0]); + exit(EXIT_FAILURE); + } + finish = clock(); + fprintf(stderr, "%.4f sec\n", (double)(finish - start) / (double)CLOCKS_PER_SEC); + + /* Write the suffix array. */ + if(fwrite(SA, sizeof(saidx_t), (size_t)n, ofp) != (size_t)n) { + fprintf(stderr, "%s: Cannot write to `%s': ", argv[0], ofname); + perror(NULL); + exit(EXIT_FAILURE); + } + if(needclose & 2) { fclose(ofp); } + + /* Deallocate memory. */ + free(SA); + free(T); + + return 0; +} diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/examples/sasearch.c b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/examples/sasearch.c new file mode 100644 index 000000000..7e5ca4fe0 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/examples/sasearch.c @@ -0,0 +1,165 @@ +/* + * sasearch.c for libdivsufsort + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#if HAVE_CONFIG_H +# include "config.h" +#endif +#include <stdio.h> +#if HAVE_STRING_H +# include <string.h> +#endif +#if HAVE_STDLIB_H +# include <stdlib.h> +#endif +#if HAVE_MEMORY_H +# include <memory.h> +#endif +#if HAVE_STDDEF_H +# include <stddef.h> +#endif +#if HAVE_STRINGS_H +# include <strings.h> +#endif +#if HAVE_SYS_TYPES_H +# include <sys/types.h> +#endif +#if HAVE_IO_H && HAVE_FCNTL_H +# include <io.h> +# include <fcntl.h> +#endif +#include <divsufsort.h> +#include "lfs.h" + + +static +void +print_help(const char *progname, int status) { + fprintf(stderr, + "sasearch, a simple SA-based full-text search tool, version %s\n", + divsufsort_version()); + fprintf(stderr, "usage: %s PATTERN FILE SAFILE\n\n", progname); + exit(status); +} + +int +main(int argc, const char *argv[]) { + FILE *fp; + const char *P; + sauchar_t *T; + saidx_t *SA; + LFS_OFF_T n; + size_t Psize; + saidx_t i, size, left; + + if((argc == 1) || + (strcmp(argv[1], "-h") == 0) || + (strcmp(argv[1], "--help") == 0)) { print_help(argv[0], EXIT_SUCCESS); } + if(argc != 4) { print_help(argv[0], EXIT_FAILURE); } + + P = argv[1]; + Psize = strlen(P); + + /* Open a file for reading. */ +#if HAVE_FOPEN_S + if(fopen_s(&fp, argv[2], "rb") != 0) { +#else + if((fp = LFS_FOPEN(argv[2], "rb")) == NULL) { +#endif + fprintf(stderr, "%s: Cannot open file `%s': ", argv[0], argv[2]); + perror(NULL); + exit(EXIT_FAILURE); + } + + /* Get the file size. */ + if(LFS_FSEEK(fp, 0, SEEK_END) == 0) { + n = LFS_FTELL(fp); + rewind(fp); + if(n < 0) { + fprintf(stderr, "%s: Cannot ftell `%s': ", argv[0], argv[2]); + perror(NULL); + exit(EXIT_FAILURE); + } + } else { + fprintf(stderr, "%s: Cannot fseek `%s': ", argv[0], argv[2]); + perror(NULL); + exit(EXIT_FAILURE); + } + + /* Allocate 5n bytes of memory. */ + T = (sauchar_t *)malloc((size_t)n * sizeof(sauchar_t)); + SA = (saidx_t *)malloc((size_t)n * sizeof(saidx_t)); + if((T == NULL) || (SA == NULL)) { + fprintf(stderr, "%s: Cannot allocate memory.\n", argv[0]); + exit(EXIT_FAILURE); + } + + /* Read n bytes of data. */ + if(fread(T, sizeof(sauchar_t), (size_t)n, fp) != (size_t)n) { + fprintf(stderr, "%s: %s `%s': ", + argv[0], + (ferror(fp) || !feof(fp)) ? "Cannot read from" : "Unexpected EOF in", + argv[2]); + perror(NULL); + exit(EXIT_FAILURE); + } + fclose(fp); + + /* Open the SA file for reading. */ +#if HAVE_FOPEN_S + if(fopen_s(&fp, argv[3], "rb") != 0) { +#else + if((fp = LFS_FOPEN(argv[3], "rb")) == NULL) { +#endif + fprintf(stderr, "%s: Cannot open file `%s': ", argv[0], argv[3]); + perror(NULL); + exit(EXIT_FAILURE); + } + + /* Read n * sizeof(saidx_t) bytes of data. */ + if(fread(SA, sizeof(saidx_t), (size_t)n, fp) != (size_t)n) { + fprintf(stderr, "%s: %s `%s': ", + argv[0], + (ferror(fp) || !feof(fp)) ? "Cannot read from" : "Unexpected EOF in", + argv[3]); + perror(NULL); + exit(EXIT_FAILURE); + } + fclose(fp); + + /* Search and print */ + size = sa_search(T, (saidx_t)n, + (const sauchar_t *)P, (saidx_t)Psize, + SA, (saidx_t)n, &left); + for(i = 0; i < size; ++i) { + fprintf(stdout, "%" PRIdSAIDX_T "\n", SA[left + i]); + } + + /* Deallocate memory. */ + free(SA); + free(T); + + return 0; +} diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/examples/suftest.c b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/examples/suftest.c new file mode 100644 index 000000000..71892ac17 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/examples/suftest.c @@ -0,0 +1,164 @@ +/* + * suftest.c for libdivsufsort + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#if HAVE_CONFIG_H +# include "config.h" +#endif +#include <stdio.h> +#if HAVE_STRING_H +# include <string.h> +#endif +#if HAVE_STDLIB_H +# include <stdlib.h> +#endif +#if HAVE_MEMORY_H +# include <memory.h> +#endif +#if HAVE_STDDEF_H +# include <stddef.h> +#endif +#if HAVE_STRINGS_H +# include <strings.h> +#endif +#if HAVE_SYS_TYPES_H +# include <sys/types.h> +#endif +#if HAVE_IO_H && HAVE_FCNTL_H +# include <io.h> +# include <fcntl.h> +#endif +#include <time.h> +#include <divsufsort.h> +#include "lfs.h" + + +static +void +print_help(const char *progname, int status) { + fprintf(stderr, + "suftest, a suffixsort tester, version %s.\n", + divsufsort_version()); + fprintf(stderr, "usage: %s FILE\n\n", progname); + exit(status); +} + +int +main(int argc, const char *argv[]) { + FILE *fp; + const char *fname; + sauchar_t *T; + saidx_t *SA; + LFS_OFF_T n; + clock_t start, finish; + saint_t needclose = 1; + + /* Check arguments. */ + if((argc == 1) || + (strcmp(argv[1], "-h") == 0) || + (strcmp(argv[1], "--help") == 0)) { print_help(argv[0], EXIT_SUCCESS); } + if(argc != 2) { print_help(argv[0], EXIT_FAILURE); } + + /* Open a file for reading. */ + if(strcmp(argv[1], "-") != 0) { +#if HAVE_FOPEN_S + if(fopen_s(&fp, fname = argv[1], "rb") != 0) { +#else + if((fp = LFS_FOPEN(fname = argv[1], "rb")) == NULL) { +#endif + fprintf(stderr, "%s: Cannot open file `%s': ", argv[0], fname); + perror(NULL); + exit(EXIT_FAILURE); + } + } else { +#if HAVE__SETMODE && HAVE__FILENO + if(_setmode(_fileno(stdin), _O_BINARY) == -1) { + fprintf(stderr, "%s: Cannot set mode: ", argv[0]); + perror(NULL); + exit(EXIT_FAILURE); + } +#endif + fp = stdin; + fname = "stdin"; + needclose = 0; + } + + /* Get the file size. */ + if(LFS_FSEEK(fp, 0, SEEK_END) == 0) { + n = LFS_FTELL(fp); + rewind(fp); + if(n < 0) { + fprintf(stderr, "%s: Cannot ftell `%s': ", argv[0], fname); + perror(NULL); + exit(EXIT_FAILURE); + } + if(0x7fffffff <= n) { + fprintf(stderr, "%s: Input file `%s' is too big.\n", argv[0], fname); + exit(EXIT_FAILURE); + } + } else { + fprintf(stderr, "%s: Cannot fseek `%s': ", argv[0], fname); + perror(NULL); + exit(EXIT_FAILURE); + } + + /* Allocate 5n bytes of memory. */ + T = (sauchar_t *)malloc((size_t)n * sizeof(sauchar_t)); + SA = (saidx_t *)malloc((size_t)n * sizeof(saidx_t)); + if((T == NULL) || (SA == NULL)) { + fprintf(stderr, "%s: Cannot allocate memory.\n", argv[0]); + exit(EXIT_FAILURE); + } + + /* Read n bytes of data. */ + if(fread(T, sizeof(sauchar_t), (size_t)n, fp) != (size_t)n) { + fprintf(stderr, "%s: %s `%s': ", + argv[0], + (ferror(fp) || !feof(fp)) ? "Cannot read from" : "Unexpected EOF in", + argv[1]); + perror(NULL); + exit(EXIT_FAILURE); + } + if(needclose & 1) { fclose(fp); } + + /* Construct the suffix array. */ + fprintf(stderr, "%s: %" PRIdOFF_T " bytes ... ", fname, n); + start = clock(); + if(divsufsort(T, SA, (saidx_t)n) != 0) { + fprintf(stderr, "%s: Cannot allocate memory.\n", argv[0]); + exit(EXIT_FAILURE); + } + finish = clock(); + fprintf(stderr, "%.4f sec\n", (double)(finish - start) / (double)CLOCKS_PER_SEC); + + /* Check the suffix array. */ + if(sufcheck(T, SA, (saidx_t)n, 1) != 0) { exit(EXIT_FAILURE); } + + /* Deallocate memory. */ + free(SA); + free(T); + + return 0; +} diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/examples/unbwt.c b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/examples/unbwt.c new file mode 100644 index 000000000..c0f19e97a --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/examples/unbwt.c @@ -0,0 +1,207 @@ +/* + * unbwt.c for libdivsufsort + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#if HAVE_CONFIG_H +# include "config.h" +#endif +#include <stdio.h> +#if HAVE_STRING_H +# include <string.h> +#endif +#if HAVE_STDLIB_H +# include <stdlib.h> +#endif +#if HAVE_MEMORY_H +# include <memory.h> +#endif +#if HAVE_STDDEF_H +# include <stddef.h> +#endif +#if HAVE_STRINGS_H +# include <strings.h> +#endif +#if HAVE_SYS_TYPES_H +# include <sys/types.h> +#endif +#if HAVE_IO_H && HAVE_FCNTL_H +# include <io.h> +# include <fcntl.h> +#endif +#include <time.h> +#include <divsufsort.h> +#include "lfs.h" + + +static +size_t +read_int(FILE *fp, saidx_t *n) { + unsigned char c[4]; + size_t m = fread(c, sizeof(unsigned char), 4, fp); + if(m == 4) { + *n = (c[0] << 0) | (c[1] << 8) | + (c[2] << 16) | (c[3] << 24); + } + return m; +} + +static +void +print_help(const char *progname, int status) { + fprintf(stderr, + "unbwt, an inverse burrows-wheeler transform program, version %s.\n", + divsufsort_version()); + fprintf(stderr, "usage: %s INFILE OUTFILE\n\n", progname); + exit(status); +} + +int +main(int argc, const char *argv[]) { + FILE *fp, *ofp; + const char *fname, *ofname; + sauchar_t *T; + saidx_t *A; + LFS_OFF_T n; + size_t m; + saidx_t pidx; + clock_t start, finish; + saint_t err, blocksize, needclose = 3; + + /* Check arguments. */ + if((argc == 1) || + (strcmp(argv[1], "-h") == 0) || + (strcmp(argv[1], "--help") == 0)) { print_help(argv[0], EXIT_SUCCESS); } + if(argc != 3) { print_help(argv[0], EXIT_FAILURE); } + + /* Open a file for reading. */ + if(strcmp(argv[1], "-") != 0) { +#if HAVE_FOPEN_S + if(fopen_s(&fp, fname = argv[1], "rb") != 0) { +#else + if((fp = LFS_FOPEN(fname = argv[1], "rb")) == NULL) { +#endif + fprintf(stderr, "%s: Cannot open file `%s': ", argv[0], fname); + perror(NULL); + exit(EXIT_FAILURE); + } + } else { +#if HAVE__SETMODE && HAVE__FILENO + if(_setmode(_fileno(stdin), _O_BINARY) == -1) { + fprintf(stderr, "%s: Cannot set mode: ", argv[0]); + perror(NULL); + exit(EXIT_FAILURE); + } +#endif + fp = stdin; + fname = "stdin"; + needclose ^= 1; + } + + /* Open a file for writing. */ + if(strcmp(argv[2], "-") != 0) { +#if HAVE_FOPEN_S + if(fopen_s(&ofp, ofname = argv[2], "wb") != 0) { +#else + if((ofp = LFS_FOPEN(ofname = argv[2], "wb")) == NULL) { +#endif + fprintf(stderr, "%s: Cannot open file `%s': ", argv[0], ofname); + perror(NULL); + exit(EXIT_FAILURE); + } + } else { +#if HAVE__SETMODE && HAVE__FILENO + if(_setmode(_fileno(stdout), _O_BINARY) == -1) { + fprintf(stderr, "%s: Cannot set mode: ", argv[0]); + perror(NULL); + exit(EXIT_FAILURE); + } +#endif + ofp = stdout; + ofname = "stdout"; + needclose ^= 2; + } + + /* Read the blocksize. */ + if(read_int(fp, &blocksize) != 4) { + fprintf(stderr, "%s: Cannot read from `%s': ", argv[0], fname); + perror(NULL); + exit(EXIT_FAILURE); + } + + /* Allocate 5blocksize bytes of memory. */ + T = (sauchar_t *)malloc(blocksize * sizeof(sauchar_t)); + A = (saidx_t *)malloc(blocksize * sizeof(saidx_t)); + if((T == NULL) || (A == NULL)) { + fprintf(stderr, "%s: Cannot allocate memory.\n", argv[0]); + exit(EXIT_FAILURE); + } + + fprintf(stderr, "UnBWT (blocksize %" PRIdSAINT_T ") ... ", blocksize); + start = clock(); + for(n = 0; (m = read_int(fp, &pidx)) != 0; n += m) { + /* Read blocksize bytes of data. */ + if((m != 4) || ((m = fread(T, sizeof(sauchar_t), blocksize, fp)) == 0)) { + fprintf(stderr, "%s: %s `%s': ", + argv[0], + (ferror(fp) || !feof(fp)) ? "Cannot read from" : "Unexpected EOF in", + fname); + perror(NULL); + exit(EXIT_FAILURE); + } + + /* Inverse Burrows-Wheeler Transform. */ + if((err = inverse_bw_transform(T, T, A, m, pidx)) != 0) { + fprintf(stderr, "%s (reverseBWT): %s.\n", + argv[0], + (err == -1) ? "Invalid data" : "Cannot allocate memory"); + exit(EXIT_FAILURE); + } + + /* Write m bytes of data. */ + if(fwrite(T, sizeof(sauchar_t), m, ofp) != m) { + fprintf(stderr, "%s: Cannot write to `%s': ", argv[0], ofname); + perror(NULL); + exit(EXIT_FAILURE); + } + } + if(ferror(fp)) { + fprintf(stderr, "%s: Cannot read from `%s': ", argv[0], fname); + perror(NULL); + exit(EXIT_FAILURE); + } + finish = clock(); + fprintf(stderr, "%" PRIdOFF_T " bytes: %.4f sec\n", + n, (double)(finish - start) / (double)CLOCKS_PER_SEC); + + /* Close files */ + if(needclose & 1) { fclose(fp); } + if(needclose & 2) { fclose(ofp); } + + /* Deallocate memory. */ + free(A); + free(T); + + return 0; +} diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/include/CMakeLists.txt b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/include/CMakeLists.txt new file mode 100644 index 000000000..37781ccb3 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/include/CMakeLists.txt @@ -0,0 +1,162 @@ +include(CheckIncludeFiles) +include(CheckIncludeFile) +include(CheckSymbolExists) +include(CheckTypeSize) +include(CheckFunctionKeywords) +include(CheckLFS) + +## Checks for header files ## +check_include_file("inttypes.h" HAVE_INTTYPES_H) +check_include_file("memory.h" HAVE_MEMORY_H) +check_include_file("stddef.h" HAVE_STDDEF_H) +check_include_file("stdint.h" HAVE_STDINT_H) +check_include_file("stdlib.h" HAVE_STDLIB_H) +check_include_file("string.h" HAVE_STRING_H) +check_include_file("strings.h" HAVE_STRINGS_H) +check_include_file("sys/types.h" HAVE_SYS_TYPES_H) +if(HAVE_INTTYPES_H) + set(INCFILE "#include <inttypes.h>") +elseif(HAVE_STDINT_H) + set(INCFILE "#include <stdint.h>") +else(HAVE_INTTYPES_H) + set(INCFILE "") +endif(HAVE_INTTYPES_H) + +## create configuration files from .cmake file ## +if(BUILD_EXAMPLES) + ## Checks for WinIO ## + if(WIN32) + check_include_file("io.h" HAVE_IO_H) + check_include_file("fcntl.h" HAVE_FCNTL_H) + check_symbol_exists("_setmode" "io.h;fcntl.h" HAVE__SETMODE) + if(NOT HAVE__SETMODE) + check_symbol_exists("setmode" "io.h;fcntl.h" HAVE_SETMODE) + endif(NOT HAVE__SETMODE) + check_symbol_exists("_fileno" "stdio.h" HAVE__FILENO) + check_symbol_exists("fopen_s" "stdio.h" HAVE_FOPEN_S) + check_symbol_exists("_O_BINARY" "fcntl.h" HAVE__O_BINARY) + endif(WIN32) + + ## Checks for large file support ## + check_lfs(WITH_LFS) + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/lfs.h.cmake" "${CMAKE_CURRENT_BINARY_DIR}/lfs.h" @ONLY) +endif(BUILD_EXAMPLES) + +## generate config.h ## +check_function_keywords("inline;__inline;__inline__;__declspec(dllexport);__declspec(dllimport)") +if(HAVE_INLINE) + set(INLINE "inline") +elseif(HAVE___INLINE) + set(INLINE "__inline") +elseif(HAVE___INLINE__) + set(INLINE "__inline__") +else(HAVE_INLINE) + set(INLINE "") +endif(HAVE_INLINE) +configure_file("${CMAKE_CURRENT_SOURCE_DIR}/config.h.cmake" "${CMAKE_CURRENT_BINARY_DIR}/config.h") + +## Checks for types ## +# sauchar_t (8bit) +check_type_size("uint8_t" UINT8_T) +if(HAVE_UINT8_T) + set(SAUCHAR_TYPE "uint8_t") +else(HAVE_UINT8_T) + check_type_size("unsigned char" SIZEOF_UNSIGNED_CHAR) + if("${SIZEOF_UNSIGNED_CHAR}" STREQUAL "1") + set(SAUCHAR_TYPE "unsigned char") + else("${SIZEOF_UNSIGNED_CHAR}" STREQUAL "1") + message(FATAL_ERROR "Cannot find unsigned 8-bit integer type") + endif("${SIZEOF_UNSIGNED_CHAR}" STREQUAL "1") +endif(HAVE_UINT8_T) +# saint_t (32bit) +check_type_size("int32_t" INT32_T) +if(HAVE_INT32_T) + set(SAINT32_TYPE "int32_t") + check_symbol_exists("PRId32" "inttypes.h" HAVE_PRID32) + if(HAVE_PRID32) + set(SAINT32_PRId "PRId32") + else(HAVE_PRID32) + set(SAINT32_PRId "\"d\"") + endif(HAVE_PRID32) +else(HAVE_INT32_T) + check_type_size("int" SIZEOF_INT) + check_type_size("long" SIZEOF_LONG) + check_type_size("short" SIZEOF_SHORT) + check_type_size("__int32" SIZEOF___INT32) + if("${SIZEOF_INT}" STREQUAL "4") + set(SAINT32_TYPE "int") + set(SAINT32_PRId "\"d\"") + elseif("${SIZEOF_LONG}" STREQUAL "4") + set(SAINT32_TYPE "long") + set(SAINT32_PRId "\"ld\"") + elseif("${SIZEOF_SHORT}" STREQUAL "4") + set(SAINT32_TYPE "short") + set(SAINT32_PRId "\"d\"") + elseif("${SIZEOF___INT32}" STREQUAL "4") + set(SAINT32_TYPE "__int32") + set(SAINT32_PRId "\"d\"") + else("${SIZEOF_INT}" STREQUAL "4") + message(FATAL_ERROR "Cannot find 32-bit integer type") + endif("${SIZEOF_INT}" STREQUAL "4") +endif(HAVE_INT32_T) +# saint64_t (64bit) +if(BUILD_DIVSUFSORT64) + check_type_size("int64_t" INT64_T) + if(HAVE_INT64_T) + set(SAINT64_TYPE "int64_t") + check_symbol_exists("PRId64" "inttypes.h" HAVE_PRID64) + if(HAVE_PRID64) + set(SAINT64_PRId "PRId64") + else(HAVE_PRID64) + set(SAINT64_PRId "\"lld\"") + endif(HAVE_PRID64) + else(HAVE_INT64_T) + check_type_size("int" SIZEOF_INT) + check_type_size("long" SIZEOF_LONG) + check_type_size("long long" SIZEOF_LONG_LONG) + check_type_size("__int64" SIZEOF___INT64) + if("${SIZEOF_INT}" STREQUAL "8") + set(SAINT64_TYPE "int") + set(SAINT64_PRId "\"d\"") + elseif("${SIZEOF_LONG}" STREQUAL "8") + set(SAINT64_TYPE "long") + set(SAINT64_PRId "\"ld\"") + elseif("${SIZEOF_LONG_LONG}" STREQUAL "8") + set(SAINT64_TYPE "long long") + set(SAINT64_PRId "\"lld\"") + elseif("${SIZEOF___INT64}" STREQUAL "8") + set(SAINT64_TYPE "__int64") + set(SAINT64_PRId "\"I64d\"") + else("${SIZEOF_INT}" STREQUAL "8") + message(SEND_ERROR "Cannot find 64-bit integer type") + set(BUILD_DIVSUFSORT64 OFF) + endif("${SIZEOF_INT}" STREQUAL "8") + endif(HAVE_INT64_T) +endif(BUILD_DIVSUFSORT64) + +## generate divsufsort.h ## +set(DIVSUFSORT_IMPORT "") +set(DIVSUFSORT_EXPORT "") +if(BUILD_SHARED_LIBS) + if(HAVE___DECLSPEC_DLLIMPORT_) + set(DIVSUFSORT_IMPORT "__declspec(dllimport)") + endif(HAVE___DECLSPEC_DLLIMPORT_) + if(HAVE___DECLSPEC_DLLEXPORT_) + set(DIVSUFSORT_EXPORT "__declspec(dllexport)") + endif(HAVE___DECLSPEC_DLLEXPORT_) +endif(BUILD_SHARED_LIBS) +set(W64BIT "") +set(SAINDEX_TYPE "${SAINT32_TYPE}") +set(SAINDEX_PRId "${SAINT32_PRId}") +set(SAINT_PRId "${SAINT32_PRId}") +configure_file("${CMAKE_CURRENT_SOURCE_DIR}/divsufsort.h.cmake" + "${CMAKE_CURRENT_BINARY_DIR}/divsufsort.h" @ONLY) +install(FILES "${CMAKE_CURRENT_BINARY_DIR}/divsufsort.h" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) +if(BUILD_DIVSUFSORT64) + set(W64BIT "64") + set(SAINDEX_TYPE "${SAINT64_TYPE}") + set(SAINDEX_PRId "${SAINT64_PRId}") + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/divsufsort.h.cmake" + "${CMAKE_CURRENT_BINARY_DIR}/divsufsort64.h" @ONLY) + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/divsufsort64.h" DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) +endif(BUILD_DIVSUFSORT64) diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/include/config.h.cmake b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/include/config.h.cmake new file mode 100644 index 000000000..6a1cf47d8 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/include/config.h.cmake @@ -0,0 +1,81 @@ +/* + * config.h for libdivsufsort + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef _CONFIG_H +#define _CONFIG_H 1 + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +/** Define to the version of this package. **/ +#cmakedefine PROJECT_VERSION_FULL "${PROJECT_VERSION_FULL}" + +/** Define to 1 if you have the header files. **/ +#cmakedefine HAVE_INTTYPES_H 1 +#cmakedefine HAVE_STDDEF_H 1 +#cmakedefine HAVE_STDINT_H 1 +#cmakedefine HAVE_STDLIB_H 1 +#cmakedefine HAVE_STRING_H 1 +#cmakedefine HAVE_STRINGS_H 1 +#cmakedefine HAVE_MEMORY_H 1 +#cmakedefine HAVE_SYS_TYPES_H 1 + +/** for WinIO **/ +#cmakedefine HAVE_IO_H 1 +#cmakedefine HAVE_FCNTL_H 1 +#cmakedefine HAVE__SETMODE 1 +#cmakedefine HAVE_SETMODE 1 +#cmakedefine HAVE__FILENO 1 +#cmakedefine HAVE_FOPEN_S 1 +#cmakedefine HAVE__O_BINARY 1 +#ifndef HAVE__SETMODE +# if HAVE_SETMODE +# define _setmode setmode +# define HAVE__SETMODE 1 +# endif +# if HAVE__SETMODE && !HAVE__O_BINARY +# define _O_BINARY 0 +# define HAVE__O_BINARY 1 +# endif +#endif + +/** for inline **/ +#ifndef INLINE +# define INLINE @INLINE@ +#endif + +/** for VC++ warning **/ +#ifdef _MSC_VER +#pragma warning(disable: 4127) +#endif + + +#ifdef __cplusplus +} /* extern "C" */ +#endif /* __cplusplus */ + +#endif /* _CONFIG_H */ diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/include/divsufsort.h.cmake b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/include/divsufsort.h.cmake new file mode 100644 index 000000000..bcaba7c64 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/include/divsufsort.h.cmake @@ -0,0 +1,180 @@ +/* + * divsufsort@W64BIT@.h for libdivsufsort@W64BIT@ + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef _DIVSUFSORT@W64BIT@_H +#define _DIVSUFSORT@W64BIT@_H 1 + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +@INCFILE@ + +#ifndef DIVSUFSORT_API +# ifdef DIVSUFSORT_BUILD_DLL +# define DIVSUFSORT_API @DIVSUFSORT_EXPORT@ +# else +# define DIVSUFSORT_API @DIVSUFSORT_IMPORT@ +# endif +#endif + +/*- Datatypes -*/ +#ifndef SAUCHAR_T +#define SAUCHAR_T +typedef @SAUCHAR_TYPE@ sauchar_t; +#endif /* SAUCHAR_T */ +#ifndef SAINT_T +#define SAINT_T +typedef @SAINT32_TYPE@ saint_t; +#endif /* SAINT_T */ +#ifndef SAIDX@W64BIT@_T +#define SAIDX@W64BIT@_T +typedef @SAINDEX_TYPE@ saidx@W64BIT@_t; +#endif /* SAIDX@W64BIT@_T */ +#ifndef PRIdSAINT_T +#define PRIdSAINT_T @SAINT_PRId@ +#endif /* PRIdSAINT_T */ +#ifndef PRIdSAIDX@W64BIT@_T +#define PRIdSAIDX@W64BIT@_T @SAINDEX_PRId@ +#endif /* PRIdSAIDX@W64BIT@_T */ + + +/*- Prototypes -*/ + +/** + * Constructs the suffix array of a given string. + * @param T[0..n-1] The input string. + * @param SA[0..n-1] The output array of suffixes. + * @param n The length of the given string. + * @return 0 if no error occurred, -1 or -2 otherwise. + */ +DIVSUFSORT_API +saint_t +divsufsort@W64BIT@(const sauchar_t *T, saidx@W64BIT@_t *SA, saidx@W64BIT@_t n); + +/** + * Constructs the burrows-wheeler transformed string of a given string. + * @param T[0..n-1] The input string. + * @param U[0..n-1] The output string. (can be T) + * @param A[0..n-1] The temporary array. (can be NULL) + * @param n The length of the given string. + * @return The primary index if no error occurred, -1 or -2 otherwise. + */ +DIVSUFSORT_API +saidx@W64BIT@_t +divbwt@W64BIT@(const sauchar_t *T, sauchar_t *U, saidx@W64BIT@_t *A, saidx@W64BIT@_t n); + +/** + * Returns the version of the divsufsort library. + * @return The version number string. + */ +DIVSUFSORT_API +const char * +divsufsort@W64BIT@_version(void); + + +/** + * Constructs the burrows-wheeler transformed string of a given string and suffix array. + * @param T[0..n-1] The input string. + * @param U[0..n-1] The output string. (can be T) + * @param SA[0..n-1] The suffix array. (can be NULL) + * @param n The length of the given string. + * @param idx The output primary index. + * @return 0 if no error occurred, -1 or -2 otherwise. + */ +DIVSUFSORT_API +saint_t +bw_transform@W64BIT@(const sauchar_t *T, sauchar_t *U, + saidx@W64BIT@_t *SA /* can NULL */, + saidx@W64BIT@_t n, saidx@W64BIT@_t *idx); + +/** + * Inverse BW-transforms a given BWTed string. + * @param T[0..n-1] The input string. + * @param U[0..n-1] The output string. (can be T) + * @param A[0..n-1] The temporary array. (can be NULL) + * @param n The length of the given string. + * @param idx The primary index. + * @return 0 if no error occurred, -1 or -2 otherwise. + */ +DIVSUFSORT_API +saint_t +inverse_bw_transform@W64BIT@(const sauchar_t *T, sauchar_t *U, + saidx@W64BIT@_t *A /* can NULL */, + saidx@W64BIT@_t n, saidx@W64BIT@_t idx); + +/** + * Checks the correctness of a given suffix array. + * @param T[0..n-1] The input string. + * @param SA[0..n-1] The input suffix array. + * @param n The length of the given string. + * @param verbose The verbose mode. + * @return 0 if no error occurred. + */ +DIVSUFSORT_API +saint_t +sufcheck@W64BIT@(const sauchar_t *T, const saidx@W64BIT@_t *SA, saidx@W64BIT@_t n, saint_t verbose); + +/** + * Search for the pattern P in the string T. + * @param T[0..Tsize-1] The input string. + * @param Tsize The length of the given string. + * @param P[0..Psize-1] The input pattern string. + * @param Psize The length of the given pattern string. + * @param SA[0..SAsize-1] The input suffix array. + * @param SAsize The length of the given suffix array. + * @param idx The output index. + * @return The count of matches if no error occurred, -1 otherwise. + */ +DIVSUFSORT_API +saidx@W64BIT@_t +sa_search@W64BIT@(const sauchar_t *T, saidx@W64BIT@_t Tsize, + const sauchar_t *P, saidx@W64BIT@_t Psize, + const saidx@W64BIT@_t *SA, saidx@W64BIT@_t SAsize, + saidx@W64BIT@_t *left); + +/** + * Search for the character c in the string T. + * @param T[0..Tsize-1] The input string. + * @param Tsize The length of the given string. + * @param SA[0..SAsize-1] The input suffix array. + * @param SAsize The length of the given suffix array. + * @param c The input character. + * @param idx The output index. + * @return The count of matches if no error occurred, -1 otherwise. + */ +DIVSUFSORT_API +saidx@W64BIT@_t +sa_simplesearch@W64BIT@(const sauchar_t *T, saidx@W64BIT@_t Tsize, + const saidx@W64BIT@_t *SA, saidx@W64BIT@_t SAsize, + saint_t c, saidx@W64BIT@_t *left); + + +#ifdef __cplusplus +} /* extern "C" */ +#endif /* __cplusplus */ + +#endif /* _DIVSUFSORT@W64BIT@_H */ diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/include/divsufsort_private.h b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/include/divsufsort_private.h new file mode 100644 index 000000000..7e261c19d --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/include/divsufsort_private.h @@ -0,0 +1,207 @@ +/* + * divsufsort_private.h for libdivsufsort + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef _DIVSUFSORT_PRIVATE_H +#define _DIVSUFSORT_PRIVATE_H 1 + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +#if HAVE_CONFIG_H +# include "config.h" +#endif +#include <assert.h> +#include <stdio.h> +#if HAVE_STRING_H +# include <string.h> +#endif +#if HAVE_STDLIB_H +# include <stdlib.h> +#endif +#if HAVE_MEMORY_H +# include <memory.h> +#endif +#if HAVE_STDDEF_H +# include <stddef.h> +#endif +#if HAVE_STRINGS_H +# include <strings.h> +#endif +#if HAVE_INTTYPES_H +# include <inttypes.h> +#else +# if HAVE_STDINT_H +# include <stdint.h> +# endif +#endif +#if defined(BUILD_DIVSUFSORT64) +# include "divsufsort64.h" +# ifndef SAIDX_T +# define SAIDX_T +# define saidx_t saidx64_t +# endif /* SAIDX_T */ +# ifndef PRIdSAIDX_T +# define PRIdSAIDX_T PRIdSAIDX64_T +# endif /* PRIdSAIDX_T */ +# define divsufsort divsufsort64 +# define divbwt divbwt64 +# define divsufsort_version divsufsort64_version +# define bw_transform bw_transform64 +# define inverse_bw_transform inverse_bw_transform64 +# define sufcheck sufcheck64 +# define sa_search sa_search64 +# define sa_simplesearch sa_simplesearch64 +# define sssort sssort64 +# define trsort trsort64 +#else +# include "divsufsort.h" +#endif + + +/*- Constants -*/ +#if !defined(UINT8_MAX) +# define UINT8_MAX (255) +#endif /* UINT8_MAX */ +#if defined(ALPHABET_SIZE) && (ALPHABET_SIZE < 1) +# undef ALPHABET_SIZE +#endif +#if !defined(ALPHABET_SIZE) +# define ALPHABET_SIZE (UINT8_MAX + 1) +#endif +/* for divsufsort.c */ +#define BUCKET_A_SIZE (ALPHABET_SIZE) +#define BUCKET_B_SIZE (ALPHABET_SIZE * ALPHABET_SIZE) +/* for sssort.c */ +#if defined(SS_INSERTIONSORT_THRESHOLD) +# if SS_INSERTIONSORT_THRESHOLD < 1 +# undef SS_INSERTIONSORT_THRESHOLD +# define SS_INSERTIONSORT_THRESHOLD (1) +# endif +#else +# define SS_INSERTIONSORT_THRESHOLD (8) +#endif +#if defined(SS_BLOCKSIZE) +# if SS_BLOCKSIZE < 0 +# undef SS_BLOCKSIZE +# define SS_BLOCKSIZE (0) +# elif 32768 <= SS_BLOCKSIZE +# undef SS_BLOCKSIZE +# define SS_BLOCKSIZE (32767) +# endif +#else +# define SS_BLOCKSIZE (1024) +#endif +/* minstacksize = log(SS_BLOCKSIZE) / log(3) * 2 */ +#if SS_BLOCKSIZE == 0 +# if defined(BUILD_DIVSUFSORT64) +# define SS_MISORT_STACKSIZE (96) +# else +# define SS_MISORT_STACKSIZE (64) +# endif +#elif SS_BLOCKSIZE <= 4096 +# define SS_MISORT_STACKSIZE (16) +#else +# define SS_MISORT_STACKSIZE (24) +#endif +#if defined(BUILD_DIVSUFSORT64) +# define SS_SMERGE_STACKSIZE (64) +#else +# define SS_SMERGE_STACKSIZE (32) +#endif +/* for trsort.c */ +#define TR_INSERTIONSORT_THRESHOLD (8) +#if defined(BUILD_DIVSUFSORT64) +# define TR_STACKSIZE (96) +#else +# define TR_STACKSIZE (64) +#endif + + +/*- Macros -*/ +#ifndef SWAP +# define SWAP(_a, _b) do { t = (_a); (_a) = (_b); (_b) = t; } while(0) +#endif /* SWAP */ +#ifndef MIN +# define MIN(_a, _b) (((_a) < (_b)) ? (_a) : (_b)) +#endif /* MIN */ +#ifndef MAX +# define MAX(_a, _b) (((_a) > (_b)) ? (_a) : (_b)) +#endif /* MAX */ +#define STACK_PUSH(_a, _b, _c, _d)\ + do {\ + assert(ssize < STACK_SIZE);\ + stack[ssize].a = (_a), stack[ssize].b = (_b),\ + stack[ssize].c = (_c), stack[ssize++].d = (_d);\ + } while(0) +#define STACK_PUSH5(_a, _b, _c, _d, _e)\ + do {\ + assert(ssize < STACK_SIZE);\ + stack[ssize].a = (_a), stack[ssize].b = (_b),\ + stack[ssize].c = (_c), stack[ssize].d = (_d), stack[ssize++].e = (_e);\ + } while(0) +#define STACK_POP(_a, _b, _c, _d)\ + do {\ + assert(0 <= ssize);\ + if(ssize == 0) { return; }\ + (_a) = stack[--ssize].a, (_b) = stack[ssize].b,\ + (_c) = stack[ssize].c, (_d) = stack[ssize].d;\ + } while(0) +#define STACK_POP5(_a, _b, _c, _d, _e)\ + do {\ + assert(0 <= ssize);\ + if(ssize == 0) { return; }\ + (_a) = stack[--ssize].a, (_b) = stack[ssize].b,\ + (_c) = stack[ssize].c, (_d) = stack[ssize].d, (_e) = stack[ssize].e;\ + } while(0) +/* for divsufsort.c */ +#define BUCKET_A(_c0) bucket_A[(_c0)] +#if ALPHABET_SIZE == 256 +#define BUCKET_B(_c0, _c1) (bucket_B[((_c1) << 8) | (_c0)]) +#define BUCKET_BSTAR(_c0, _c1) (bucket_B[((_c0) << 8) | (_c1)]) +#else +#define BUCKET_B(_c0, _c1) (bucket_B[(_c1) * ALPHABET_SIZE + (_c0)]) +#define BUCKET_BSTAR(_c0, _c1) (bucket_B[(_c0) * ALPHABET_SIZE + (_c1)]) +#endif + + +/*- Private Prototypes -*/ +/* sssort.c */ +void +sssort(const sauchar_t *Td, const saidx_t *PA, + saidx_t *first, saidx_t *last, + saidx_t *buf, saidx_t bufsize, + saidx_t depth, saidx_t n, saint_t lastsuffix); +/* trsort.c */ +void +trsort(saidx_t *ISA, saidx_t *SA, saidx_t n, saidx_t depth); + + +#ifdef __cplusplus +} /* extern "C" */ +#endif /* __cplusplus */ + +#endif /* _DIVSUFSORT_PRIVATE_H */ diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/include/lfs.h.cmake b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/include/lfs.h.cmake new file mode 100644 index 000000000..d5b84a842 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/include/lfs.h.cmake @@ -0,0 +1,56 @@ +/* + * lfs.h for libdivsufsort + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#ifndef _LFS_H +#define _LFS_H 1 + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +#ifndef __STRICT_ANSI__ +# define LFS_OFF_T @LFS_OFF_T@ +# define LFS_FOPEN @LFS_FOPEN@ +# define LFS_FTELL @LFS_FTELL@ +# define LFS_FSEEK @LFS_FSEEK@ +# define LFS_PRId @LFS_PRID@ +#else +# define LFS_OFF_T long +# define LFS_FOPEN fopen +# define LFS_FTELL ftell +# define LFS_FSEEK fseek +# define LFS_PRId "ld" +#endif +#ifndef PRIdOFF_T +# define PRIdOFF_T LFS_PRId +#endif + + +#ifdef __cplusplus +} /* extern "C" */ +#endif /* __cplusplus */ + +#endif /* _LFS_H */ diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/lib/CMakeLists.txt b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/lib/CMakeLists.txt new file mode 100644 index 000000000..abc90e61d --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/lib/CMakeLists.txt @@ -0,0 +1,31 @@ +include_directories("${CMAKE_CURRENT_SOURCE_DIR}/../include" + "${CMAKE_CURRENT_BINARY_DIR}/../include") + +set(divsufsort_SRCS divsufsort.c sssort.c trsort.c utils.c) + +## libdivsufsort ## +add_library(divsufsort ${divsufsort_SRCS}) +install(TARGETS divsufsort + RUNTIME DESTINATION ${CMAKE_INSTALL_RUNTIMEDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) +set_target_properties(divsufsort PROPERTIES + VERSION "${LIBRARY_VERSION}" + SOVERSION "${LIBRARY_SOVERSION}" + DEFINE_SYMBOL DIVSUFSORT_BUILD_DLL + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/../examples") + +## libdivsufsort64 ## +if(BUILD_DIVSUFSORT64) + add_library(divsufsort64 ${divsufsort_SRCS}) + install(TARGETS divsufsort64 + RUNTIME DESTINATION ${CMAKE_INSTALL_RUNTIMEDIR} + LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) + set_target_properties(divsufsort64 PROPERTIES + VERSION "${LIBRARY_VERSION}" + SOVERSION "${LIBRARY_SOVERSION}" + DEFINE_SYMBOL DIVSUFSORT_BUILD_DLL + COMPILE_FLAGS "-DBUILD_DIVSUFSORT64" + RUNTIME_OUTPUT_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/../examples") +endif(BUILD_DIVSUFSORT64) diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/lib/divsufsort.c b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/lib/divsufsort.c new file mode 100644 index 000000000..9f64b4f48 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/lib/divsufsort.c @@ -0,0 +1,398 @@ +/* + * divsufsort.c for libdivsufsort + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "divsufsort_private.h" +#ifdef _OPENMP +# include <omp.h> +#endif + + +/*- Private Functions -*/ + +/* Sorts suffixes of type B*. */ +static +saidx_t +sort_typeBstar(const sauchar_t *T, saidx_t *SA, + saidx_t *bucket_A, saidx_t *bucket_B, + saidx_t n) { + saidx_t *PAb, *ISAb, *buf; +#ifdef _OPENMP + saidx_t *curbuf; + saidx_t l; +#endif + saidx_t i, j, k, t, m, bufsize; + saint_t c0, c1; +#ifdef _OPENMP + saint_t d0, d1; + int tmp; +#endif + + /* Initialize bucket arrays. */ + for(i = 0; i < BUCKET_A_SIZE; ++i) { bucket_A[i] = 0; } + for(i = 0; i < BUCKET_B_SIZE; ++i) { bucket_B[i] = 0; } + + /* Count the number of occurrences of the first one or two characters of each + type A, B and B* suffix. Moreover, store the beginning position of all + type B* suffixes into the array SA. */ + for(i = n - 1, m = n, c0 = T[n - 1]; 0 <= i;) { + /* type A suffix. */ + do { ++BUCKET_A(c1 = c0); } while((0 <= --i) && ((c0 = T[i]) >= c1)); + if(0 <= i) { + /* type B* suffix. */ + ++BUCKET_BSTAR(c0, c1); + SA[--m] = i; + /* type B suffix. */ + for(--i, c1 = c0; (0 <= i) && ((c0 = T[i]) <= c1); --i, c1 = c0) { + ++BUCKET_B(c0, c1); + } + } + } + m = n - m; +/* +note: + A type B* suffix is lexicographically smaller than a type B suffix that + begins with the same first two characters. +*/ + + /* Calculate the index of start/end point of each bucket. */ + for(c0 = 0, i = 0, j = 0; c0 < ALPHABET_SIZE; ++c0) { + t = i + BUCKET_A(c0); + BUCKET_A(c0) = i + j; /* start point */ + i = t + BUCKET_B(c0, c0); + for(c1 = c0 + 1; c1 < ALPHABET_SIZE; ++c1) { + j += BUCKET_BSTAR(c0, c1); + BUCKET_BSTAR(c0, c1) = j; /* end point */ + i += BUCKET_B(c0, c1); + } + } + + if(0 < m) { + /* Sort the type B* suffixes by their first two characters. */ + PAb = SA + n - m; ISAb = SA + m; + for(i = m - 2; 0 <= i; --i) { + t = PAb[i], c0 = T[t], c1 = T[t + 1]; + SA[--BUCKET_BSTAR(c0, c1)] = i; + } + t = PAb[m - 1], c0 = T[t], c1 = T[t + 1]; + SA[--BUCKET_BSTAR(c0, c1)] = m - 1; + + /* Sort the type B* substrings using sssort. */ +#ifdef _OPENMP + tmp = omp_get_max_threads(); + buf = SA + m, bufsize = (n - (2 * m)) / tmp; + c0 = ALPHABET_SIZE - 2, c1 = ALPHABET_SIZE - 1, j = m; +#pragma omp parallel default(shared) private(curbuf, k, l, d0, d1, tmp) + { + tmp = omp_get_thread_num(); + curbuf = buf + tmp * bufsize; + k = 0; + for(;;) { + #pragma omp critical(sssort_lock) + { + if(0 < (l = j)) { + d0 = c0, d1 = c1; + do { + k = BUCKET_BSTAR(d0, d1); + if(--d1 <= d0) { + d1 = ALPHABET_SIZE - 1; + if(--d0 < 0) { break; } + } + } while(((l - k) <= 1) && (0 < (l = k))); + c0 = d0, c1 = d1, j = k; + } + } + if(l == 0) { break; } + sssort(T, PAb, SA + k, SA + l, + curbuf, bufsize, 2, n, *(SA + k) == (m - 1)); + } + } +#else + buf = SA + m, bufsize = n - (2 * m); + for(c0 = ALPHABET_SIZE - 2, j = m; 0 < j; --c0) { + for(c1 = ALPHABET_SIZE - 1; c0 < c1; j = i, --c1) { + i = BUCKET_BSTAR(c0, c1); + if(1 < (j - i)) { + sssort(T, PAb, SA + i, SA + j, + buf, bufsize, 2, n, *(SA + i) == (m - 1)); + } + } + } +#endif + + /* Compute ranks of type B* substrings. */ + for(i = m - 1; 0 <= i; --i) { + if(0 <= SA[i]) { + j = i; + do { ISAb[SA[i]] = i; } while((0 <= --i) && (0 <= SA[i])); + SA[i + 1] = i - j; + if(i <= 0) { break; } + } + j = i; + do { ISAb[SA[i] = ~SA[i]] = j; } while(SA[--i] < 0); + ISAb[SA[i]] = j; + } + + /* Construct the inverse suffix array of type B* suffixes using trsort. */ + trsort(ISAb, SA, m, 1); + + /* Set the sorted order of tyoe B* suffixes. */ + for(i = n - 1, j = m, c0 = T[n - 1]; 0 <= i;) { + for(--i, c1 = c0; (0 <= i) && ((c0 = T[i]) >= c1); --i, c1 = c0) { } + if(0 <= i) { + t = i; + for(--i, c1 = c0; (0 <= i) && ((c0 = T[i]) <= c1); --i, c1 = c0) { } + SA[ISAb[--j]] = ((t == 0) || (1 < (t - i))) ? t : ~t; + } + } + + /* Calculate the index of start/end point of each bucket. */ + BUCKET_B(ALPHABET_SIZE - 1, ALPHABET_SIZE - 1) = n; /* end point */ + for(c0 = ALPHABET_SIZE - 2, k = m - 1; 0 <= c0; --c0) { + i = BUCKET_A(c0 + 1) - 1; + for(c1 = ALPHABET_SIZE - 1; c0 < c1; --c1) { + t = i - BUCKET_B(c0, c1); + BUCKET_B(c0, c1) = i; /* end point */ + + /* Move all type B* suffixes to the correct position. */ + for(i = t, j = BUCKET_BSTAR(c0, c1); + j <= k; + --i, --k) { SA[i] = SA[k]; } + } + BUCKET_BSTAR(c0, c0 + 1) = i - BUCKET_B(c0, c0) + 1; /* start point */ + BUCKET_B(c0, c0) = i; /* end point */ + } + } + + return m; +} + +/* Constructs the suffix array by using the sorted order of type B* suffixes. */ +static +void +construct_SA(const sauchar_t *T, saidx_t *SA, + saidx_t *bucket_A, saidx_t *bucket_B, + saidx_t n, saidx_t m) { + saidx_t *i, *j, *k; + saidx_t s; + saint_t c0, c1, c2; + + if(0 < m) { + /* Construct the sorted order of type B suffixes by using + the sorted order of type B* suffixes. */ + for(c1 = ALPHABET_SIZE - 2; 0 <= c1; --c1) { + /* Scan the suffix array from right to left. */ + for(i = SA + BUCKET_BSTAR(c1, c1 + 1), + j = SA + BUCKET_A(c1 + 1) - 1, k = NULL, c2 = -1; + i <= j; + --j) { + if(0 < (s = *j)) { + assert(T[s] == c1); + assert(((s + 1) < n) && (T[s] <= T[s + 1])); + assert(T[s - 1] <= T[s]); + *j = ~s; + c0 = T[--s]; + if((0 < s) && (T[s - 1] > c0)) { s = ~s; } + if(c0 != c2) { + if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; } + k = SA + BUCKET_B(c2 = c0, c1); + } + assert(k < j); + *k-- = s; + } else { + assert(((s == 0) && (T[s] == c1)) || (s < 0)); + *j = ~s; + } + } + } + } + + /* Construct the suffix array by using + the sorted order of type B suffixes. */ + k = SA + BUCKET_A(c2 = T[n - 1]); + *k++ = (T[n - 2] < c2) ? ~(n - 1) : (n - 1); + /* Scan the suffix array from left to right. */ + for(i = SA, j = SA + n; i < j; ++i) { + if(0 < (s = *i)) { + assert(T[s - 1] >= T[s]); + c0 = T[--s]; + if((s == 0) || (T[s - 1] < c0)) { s = ~s; } + if(c0 != c2) { + BUCKET_A(c2) = k - SA; + k = SA + BUCKET_A(c2 = c0); + } + assert(i < k); + *k++ = s; + } else { + assert(s < 0); + *i = ~s; + } + } +} + +/* Constructs the burrows-wheeler transformed string directly + by using the sorted order of type B* suffixes. */ +static +saidx_t +construct_BWT(const sauchar_t *T, saidx_t *SA, + saidx_t *bucket_A, saidx_t *bucket_B, + saidx_t n, saidx_t m) { + saidx_t *i, *j, *k, *orig; + saidx_t s; + saint_t c0, c1, c2; + + if(0 < m) { + /* Construct the sorted order of type B suffixes by using + the sorted order of type B* suffixes. */ + for(c1 = ALPHABET_SIZE - 2; 0 <= c1; --c1) { + /* Scan the suffix array from right to left. */ + for(i = SA + BUCKET_BSTAR(c1, c1 + 1), + j = SA + BUCKET_A(c1 + 1) - 1, k = NULL, c2 = -1; + i <= j; + --j) { + if(0 < (s = *j)) { + assert(T[s] == c1); + assert(((s + 1) < n) && (T[s] <= T[s + 1])); + assert(T[s - 1] <= T[s]); + c0 = T[--s]; + *j = ~((saidx_t)c0); + if((0 < s) && (T[s - 1] > c0)) { s = ~s; } + if(c0 != c2) { + if(0 <= c2) { BUCKET_B(c2, c1) = k - SA; } + k = SA + BUCKET_B(c2 = c0, c1); + } + assert(k < j); + *k-- = s; + } else if(s != 0) { + *j = ~s; +#ifndef NDEBUG + } else { + assert(T[s] == c1); +#endif + } + } + } + } + + /* Construct the BWTed string by using + the sorted order of type B suffixes. */ + k = SA + BUCKET_A(c2 = T[n - 1]); + *k++ = (T[n - 2] < c2) ? ~((saidx_t)T[n - 2]) : (n - 1); + /* Scan the suffix array from left to right. */ + for(i = SA, j = SA + n, orig = SA; i < j; ++i) { + if(0 < (s = *i)) { + assert(T[s - 1] >= T[s]); + c0 = T[--s]; + *i = c0; + if((0 < s) && (T[s - 1] < c0)) { s = ~((saidx_t)T[s - 1]); } + if(c0 != c2) { + BUCKET_A(c2) = k - SA; + k = SA + BUCKET_A(c2 = c0); + } + assert(i < k); + *k++ = s; + } else if(s != 0) { + *i = ~s; + } else { + orig = i; + } + } + + return orig - SA; +} + + +/*---------------------------------------------------------------------------*/ + +/*- Function -*/ + +saint_t +divsufsort(const sauchar_t *T, saidx_t *SA, saidx_t n) { + saidx_t *bucket_A, *bucket_B; + saidx_t m; + saint_t err = 0; + + /* Check arguments. */ + if((T == NULL) || (SA == NULL) || (n < 0)) { return -1; } + else if(n == 0) { return 0; } + else if(n == 1) { SA[0] = 0; return 0; } + else if(n == 2) { m = (T[0] < T[1]); SA[m ^ 1] = 0, SA[m] = 1; return 0; } + + bucket_A = (saidx_t *)malloc(BUCKET_A_SIZE * sizeof(saidx_t)); + bucket_B = (saidx_t *)malloc(BUCKET_B_SIZE * sizeof(saidx_t)); + + /* Suffixsort. */ + if((bucket_A != NULL) && (bucket_B != NULL)) { + m = sort_typeBstar(T, SA, bucket_A, bucket_B, n); + construct_SA(T, SA, bucket_A, bucket_B, n, m); + } else { + err = -2; + } + + free(bucket_B); + free(bucket_A); + + return err; +} + +saidx_t +divbwt(const sauchar_t *T, sauchar_t *U, saidx_t *A, saidx_t n) { + saidx_t *B; + saidx_t *bucket_A, *bucket_B; + saidx_t m, pidx, i; + + /* Check arguments. */ + if((T == NULL) || (U == NULL) || (n < 0)) { return -1; } + else if(n <= 1) { if(n == 1) { U[0] = T[0]; } return n; } + + if((B = A) == NULL) { B = (saidx_t *)malloc((size_t)(n + 1) * sizeof(saidx_t)); } + bucket_A = (saidx_t *)malloc(BUCKET_A_SIZE * sizeof(saidx_t)); + bucket_B = (saidx_t *)malloc(BUCKET_B_SIZE * sizeof(saidx_t)); + + /* Burrows-Wheeler Transform. */ + if((B != NULL) && (bucket_A != NULL) && (bucket_B != NULL)) { + m = sort_typeBstar(T, B, bucket_A, bucket_B, n); + pidx = construct_BWT(T, B, bucket_A, bucket_B, n, m); + + /* Copy to output string. */ + U[0] = T[n - 1]; + for(i = 0; i < pidx; ++i) { U[i + 1] = (sauchar_t)B[i]; } + for(i += 1; i < n; ++i) { U[i] = (sauchar_t)B[i]; } + pidx += 1; + } else { + pidx = -2; + } + + free(bucket_B); + free(bucket_A); + if(A == NULL) { free(B); } + + return pidx; +} + +const char * +divsufsort_version(void) { + return PROJECT_VERSION_FULL; +} diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/lib/sssort.c b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/lib/sssort.c new file mode 100644 index 000000000..4a18fd2ab --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/lib/sssort.c @@ -0,0 +1,815 @@ +/* + * sssort.c for libdivsufsort + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "divsufsort_private.h" + + +/*- Private Functions -*/ + +static const saint_t lg_table[256]= { + -1,0,1,1,2,2,2,2,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4, + 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, + 6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6, + 6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7 +}; + +#if (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE) + +static INLINE +saint_t +ss_ilg(saidx_t n) { +#if SS_BLOCKSIZE == 0 +# if defined(BUILD_DIVSUFSORT64) + return (n >> 32) ? + ((n >> 48) ? + ((n >> 56) ? + 56 + lg_table[(n >> 56) & 0xff] : + 48 + lg_table[(n >> 48) & 0xff]) : + ((n >> 40) ? + 40 + lg_table[(n >> 40) & 0xff] : + 32 + lg_table[(n >> 32) & 0xff])) : + ((n & 0xffff0000) ? + ((n & 0xff000000) ? + 24 + lg_table[(n >> 24) & 0xff] : + 16 + lg_table[(n >> 16) & 0xff]) : + ((n & 0x0000ff00) ? + 8 + lg_table[(n >> 8) & 0xff] : + 0 + lg_table[(n >> 0) & 0xff])); +# else + return (n & 0xffff0000) ? + ((n & 0xff000000) ? + 24 + lg_table[(n >> 24) & 0xff] : + 16 + lg_table[(n >> 16) & 0xff]) : + ((n & 0x0000ff00) ? + 8 + lg_table[(n >> 8) & 0xff] : + 0 + lg_table[(n >> 0) & 0xff]); +# endif +#elif SS_BLOCKSIZE < 256 + return lg_table[n]; +#else + return (n & 0xff00) ? + 8 + lg_table[(n >> 8) & 0xff] : + 0 + lg_table[(n >> 0) & 0xff]; +#endif +} + +#endif /* (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE) */ + +#if SS_BLOCKSIZE != 0 + +static const saint_t sqq_table[256] = { + 0, 16, 22, 27, 32, 35, 39, 42, 45, 48, 50, 53, 55, 57, 59, 61, + 64, 65, 67, 69, 71, 73, 75, 76, 78, 80, 81, 83, 84, 86, 87, 89, + 90, 91, 93, 94, 96, 97, 98, 99, 101, 102, 103, 104, 106, 107, 108, 109, +110, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, +128, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, +143, 144, 144, 145, 146, 147, 148, 149, 150, 150, 151, 152, 153, 154, 155, 155, +156, 157, 158, 159, 160, 160, 161, 162, 163, 163, 164, 165, 166, 167, 167, 168, +169, 170, 170, 171, 172, 173, 173, 174, 175, 176, 176, 177, 178, 178, 179, 180, +181, 181, 182, 183, 183, 184, 185, 185, 186, 187, 187, 188, 189, 189, 190, 191, +192, 192, 193, 193, 194, 195, 195, 196, 197, 197, 198, 199, 199, 200, 201, 201, +202, 203, 203, 204, 204, 205, 206, 206, 207, 208, 208, 209, 209, 210, 211, 211, +212, 212, 213, 214, 214, 215, 215, 216, 217, 217, 218, 218, 219, 219, 220, 221, +221, 222, 222, 223, 224, 224, 225, 225, 226, 226, 227, 227, 228, 229, 229, 230, +230, 231, 231, 232, 232, 233, 234, 234, 235, 235, 236, 236, 237, 237, 238, 238, +239, 240, 240, 241, 241, 242, 242, 243, 243, 244, 244, 245, 245, 246, 246, 247, +247, 248, 248, 249, 249, 250, 250, 251, 251, 252, 252, 253, 253, 254, 254, 255 +}; + +static INLINE +saidx_t +ss_isqrt(saidx_t x) { + saidx_t y, e; + + if(x >= (SS_BLOCKSIZE * SS_BLOCKSIZE)) { return SS_BLOCKSIZE; } + e = (x & 0xffff0000) ? + ((x & 0xff000000) ? + 24 + lg_table[(x >> 24) & 0xff] : + 16 + lg_table[(x >> 16) & 0xff]) : + ((x & 0x0000ff00) ? + 8 + lg_table[(x >> 8) & 0xff] : + 0 + lg_table[(x >> 0) & 0xff]); + + if(e >= 16) { + y = sqq_table[x >> ((e - 6) - (e & 1))] << ((e >> 1) - 7); + if(e >= 24) { y = (y + 1 + x / y) >> 1; } + y = (y + 1 + x / y) >> 1; + } else if(e >= 8) { + y = (sqq_table[x >> ((e - 6) - (e & 1))] >> (7 - (e >> 1))) + 1; + } else { + return sqq_table[x] >> 4; + } + + return (x < (y * y)) ? y - 1 : y; +} + +#endif /* SS_BLOCKSIZE != 0 */ + + +/*---------------------------------------------------------------------------*/ + +/* Compares two suffixes. */ +static INLINE +saint_t +ss_compare(const sauchar_t *T, + const saidx_t *p1, const saidx_t *p2, + saidx_t depth) { + const sauchar_t *U1, *U2, *U1n, *U2n; + + for(U1 = T + depth + *p1, + U2 = T + depth + *p2, + U1n = T + *(p1 + 1) + 2, + U2n = T + *(p2 + 1) + 2; + (U1 < U1n) && (U2 < U2n) && (*U1 == *U2); + ++U1, ++U2) { + } + + return U1 < U1n ? + (U2 < U2n ? *U1 - *U2 : 1) : + (U2 < U2n ? -1 : 0); +} + + +/*---------------------------------------------------------------------------*/ + +#if (SS_BLOCKSIZE != 1) && (SS_INSERTIONSORT_THRESHOLD != 1) + +/* Insertionsort for small size groups */ +static +void +ss_insertionsort(const sauchar_t *T, const saidx_t *PA, + saidx_t *first, saidx_t *last, saidx_t depth) { + saidx_t *i, *j; + saidx_t t; + saint_t r; + + for(i = last - 2; first <= i; --i) { + for(t = *i, j = i + 1; 0 < (r = ss_compare(T, PA + t, PA + *j, depth));) { + do { *(j - 1) = *j; } while((++j < last) && (*j < 0)); + if(last <= j) { break; } + } + if(r == 0) { *j = ~*j; } + *(j - 1) = t; + } +} + +#endif /* (SS_BLOCKSIZE != 1) && (SS_INSERTIONSORT_THRESHOLD != 1) */ + + +/*---------------------------------------------------------------------------*/ + +#if (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE) + +static INLINE +void +ss_fixdown(const sauchar_t *Td, const saidx_t *PA, + saidx_t *SA, saidx_t i, saidx_t size) { + saidx_t j, k; + saidx_t v; + saint_t c, d, e; + + for(v = SA[i], c = Td[PA[v]]; (j = 2 * i + 1) < size; SA[i] = SA[k], i = k) { + d = Td[PA[SA[k = j++]]]; + if(d < (e = Td[PA[SA[j]]])) { k = j; d = e; } + if(d <= c) { break; } + } + SA[i] = v; +} + +/* Simple top-down heapsort. */ +static +void +ss_heapsort(const sauchar_t *Td, const saidx_t *PA, saidx_t *SA, saidx_t size) { + saidx_t i, m; + saidx_t t; + + m = size; + if((size % 2) == 0) { + m--; + if(Td[PA[SA[m / 2]]] < Td[PA[SA[m]]]) { SWAP(SA[m], SA[m / 2]); } + } + + for(i = m / 2 - 1; 0 <= i; --i) { ss_fixdown(Td, PA, SA, i, m); } + if((size % 2) == 0) { SWAP(SA[0], SA[m]); ss_fixdown(Td, PA, SA, 0, m); } + for(i = m - 1; 0 < i; --i) { + t = SA[0], SA[0] = SA[i]; + ss_fixdown(Td, PA, SA, 0, i); + SA[i] = t; + } +} + + +/*---------------------------------------------------------------------------*/ + +/* Returns the median of three elements. */ +static INLINE +saidx_t * +ss_median3(const sauchar_t *Td, const saidx_t *PA, + saidx_t *v1, saidx_t *v2, saidx_t *v3) { + saidx_t *t; + if(Td[PA[*v1]] > Td[PA[*v2]]) { SWAP(v1, v2); } + if(Td[PA[*v2]] > Td[PA[*v3]]) { + if(Td[PA[*v1]] > Td[PA[*v3]]) { return v1; } + else { return v3; } + } + return v2; +} + +/* Returns the median of five elements. */ +static INLINE +saidx_t * +ss_median5(const sauchar_t *Td, const saidx_t *PA, + saidx_t *v1, saidx_t *v2, saidx_t *v3, saidx_t *v4, saidx_t *v5) { + saidx_t *t; + if(Td[PA[*v2]] > Td[PA[*v3]]) { SWAP(v2, v3); } + if(Td[PA[*v4]] > Td[PA[*v5]]) { SWAP(v4, v5); } + if(Td[PA[*v2]] > Td[PA[*v4]]) { SWAP(v2, v4); SWAP(v3, v5); } + if(Td[PA[*v1]] > Td[PA[*v3]]) { SWAP(v1, v3); } + if(Td[PA[*v1]] > Td[PA[*v4]]) { SWAP(v1, v4); SWAP(v3, v5); } + if(Td[PA[*v3]] > Td[PA[*v4]]) { return v4; } + return v3; +} + +/* Returns the pivot element. */ +static INLINE +saidx_t * +ss_pivot(const sauchar_t *Td, const saidx_t *PA, saidx_t *first, saidx_t *last) { + saidx_t *middle; + saidx_t t; + + t = last - first; + middle = first + t / 2; + + if(t <= 512) { + if(t <= 32) { + return ss_median3(Td, PA, first, middle, last - 1); + } else { + t >>= 2; + return ss_median5(Td, PA, first, first + t, middle, last - 1 - t, last - 1); + } + } + t >>= 3; + first = ss_median3(Td, PA, first, first + t, first + (t << 1)); + middle = ss_median3(Td, PA, middle - t, middle, middle + t); + last = ss_median3(Td, PA, last - 1 - (t << 1), last - 1 - t, last - 1); + return ss_median3(Td, PA, first, middle, last); +} + + +/*---------------------------------------------------------------------------*/ + +/* Binary partition for substrings. */ +static INLINE +saidx_t * +ss_partition(const saidx_t *PA, + saidx_t *first, saidx_t *last, saidx_t depth) { + saidx_t *a, *b; + saidx_t t; + for(a = first - 1, b = last;;) { + for(; (++a < b) && ((PA[*a] + depth) >= (PA[*a + 1] + 1));) { *a = ~*a; } + for(; (a < --b) && ((PA[*b] + depth) < (PA[*b + 1] + 1));) { } + if(b <= a) { break; } + t = ~*b; + *b = *a; + *a = t; + } + if(first < a) { *first = ~*first; } + return a; +} + +/* Multikey introsort for medium size groups. */ +static +void +ss_mintrosort(const sauchar_t *T, const saidx_t *PA, + saidx_t *first, saidx_t *last, + saidx_t depth) { +#define STACK_SIZE SS_MISORT_STACKSIZE + struct { saidx_t *a, *b, c; saint_t d; } stack[STACK_SIZE]; + const sauchar_t *Td; + saidx_t *a, *b, *c, *d, *e, *f; + saidx_t s, t; + saint_t ssize; + saint_t limit; + saint_t v, x = 0; + + for(ssize = 0, limit = ss_ilg(last - first);;) { + + if((last - first) <= SS_INSERTIONSORT_THRESHOLD) { +#if 1 < SS_INSERTIONSORT_THRESHOLD + if(1 < (last - first)) { ss_insertionsort(T, PA, first, last, depth); } +#endif + STACK_POP(first, last, depth, limit); + continue; + } + + Td = T + depth; + if(limit-- == 0) { ss_heapsort(Td, PA, first, last - first); } + if(limit < 0) { + for(a = first + 1, v = Td[PA[*first]]; a < last; ++a) { + if((x = Td[PA[*a]]) != v) { + if(1 < (a - first)) { break; } + v = x; + first = a; + } + } + if(Td[PA[*first] - 1] < v) { + first = ss_partition(PA, first, a, depth); + } + if((a - first) <= (last - a)) { + if(1 < (a - first)) { + STACK_PUSH(a, last, depth, -1); + last = a, depth += 1, limit = ss_ilg(a - first); + } else { + first = a, limit = -1; + } + } else { + if(1 < (last - a)) { + STACK_PUSH(first, a, depth + 1, ss_ilg(a - first)); + first = a, limit = -1; + } else { + last = a, depth += 1, limit = ss_ilg(a - first); + } + } + continue; + } + + /* choose pivot */ + a = ss_pivot(Td, PA, first, last); + v = Td[PA[*a]]; + SWAP(*first, *a); + + /* partition */ + for(b = first; (++b < last) && ((x = Td[PA[*b]]) == v);) { } + if(((a = b) < last) && (x < v)) { + for(; (++b < last) && ((x = Td[PA[*b]]) <= v);) { + if(x == v) { SWAP(*b, *a); ++a; } + } + } + for(c = last; (b < --c) && ((x = Td[PA[*c]]) == v);) { } + if((b < (d = c)) && (x > v)) { + for(; (b < --c) && ((x = Td[PA[*c]]) >= v);) { + if(x == v) { SWAP(*c, *d); --d; } + } + } + for(; b < c;) { + SWAP(*b, *c); + for(; (++b < c) && ((x = Td[PA[*b]]) <= v);) { + if(x == v) { SWAP(*b, *a); ++a; } + } + for(; (b < --c) && ((x = Td[PA[*c]]) >= v);) { + if(x == v) { SWAP(*c, *d); --d; } + } + } + + if(a <= d) { + c = b - 1; + + if((s = a - first) > (t = b - a)) { s = t; } + for(e = first, f = b - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); } + if((s = d - c) > (t = last - d - 1)) { s = t; } + for(e = b, f = last - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); } + + a = first + (b - a), c = last - (d - c); + b = (v <= Td[PA[*a] - 1]) ? a : ss_partition(PA, a, c, depth); + + if((a - first) <= (last - c)) { + if((last - c) <= (c - b)) { + STACK_PUSH(b, c, depth + 1, ss_ilg(c - b)); + STACK_PUSH(c, last, depth, limit); + last = a; + } else if((a - first) <= (c - b)) { + STACK_PUSH(c, last, depth, limit); + STACK_PUSH(b, c, depth + 1, ss_ilg(c - b)); + last = a; + } else { + STACK_PUSH(c, last, depth, limit); + STACK_PUSH(first, a, depth, limit); + first = b, last = c, depth += 1, limit = ss_ilg(c - b); + } + } else { + if((a - first) <= (c - b)) { + STACK_PUSH(b, c, depth + 1, ss_ilg(c - b)); + STACK_PUSH(first, a, depth, limit); + first = c; + } else if((last - c) <= (c - b)) { + STACK_PUSH(first, a, depth, limit); + STACK_PUSH(b, c, depth + 1, ss_ilg(c - b)); + first = c; + } else { + STACK_PUSH(first, a, depth, limit); + STACK_PUSH(c, last, depth, limit); + first = b, last = c, depth += 1, limit = ss_ilg(c - b); + } + } + } else { + limit += 1; + if(Td[PA[*first] - 1] < v) { + first = ss_partition(PA, first, last, depth); + limit = ss_ilg(last - first); + } + depth += 1; + } + } +#undef STACK_SIZE +} + +#endif /* (SS_BLOCKSIZE == 0) || (SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE) */ + + +/*---------------------------------------------------------------------------*/ + +#if SS_BLOCKSIZE != 0 + +static INLINE +void +ss_blockswap(saidx_t *a, saidx_t *b, saidx_t n) { + saidx_t t; + for(; 0 < n; --n, ++a, ++b) { + t = *a, *a = *b, *b = t; + } +} + +static INLINE +void +ss_rotate(saidx_t *first, saidx_t *middle, saidx_t *last) { + saidx_t *a, *b, t; + saidx_t l, r; + l = middle - first, r = last - middle; + for(; (0 < l) && (0 < r);) { + if(l == r) { ss_blockswap(first, middle, l); break; } + if(l < r) { + a = last - 1, b = middle - 1; + t = *a; + do { + *a-- = *b, *b-- = *a; + if(b < first) { + *a = t; + last = a; + if((r -= l + 1) <= l) { break; } + a -= 1, b = middle - 1; + t = *a; + } + } while(1); + } else { + a = first, b = middle; + t = *a; + do { + *a++ = *b, *b++ = *a; + if(last <= b) { + *a = t; + first = a + 1; + if((l -= r + 1) <= r) { break; } + a += 1, b = middle; + t = *a; + } + } while(1); + } + } +} + + +/*---------------------------------------------------------------------------*/ + +static +void +ss_inplacemerge(const sauchar_t *T, const saidx_t *PA, + saidx_t *first, saidx_t *middle, saidx_t *last, + saidx_t depth) { + const saidx_t *p; + saidx_t *a, *b; + saidx_t len, half; + saint_t q, r; + saint_t x; + + for(;;) { + if(*(last - 1) < 0) { x = 1; p = PA + ~*(last - 1); } + else { x = 0; p = PA + *(last - 1); } + for(a = first, len = middle - first, half = len >> 1, r = -1; + 0 < len; + len = half, half >>= 1) { + b = a + half; + q = ss_compare(T, PA + ((0 <= *b) ? *b : ~*b), p, depth); + if(q < 0) { + a = b + 1; + half -= (len & 1) ^ 1; + } else { + r = q; + } + } + if(a < middle) { + if(r == 0) { *a = ~*a; } + ss_rotate(a, middle, last); + last -= middle - a; + middle = a; + if(first == middle) { break; } + } + --last; + if(x != 0) { while(*--last < 0) { } } + if(middle == last) { break; } + } +} + + +/*---------------------------------------------------------------------------*/ + +/* Merge-forward with internal buffer. */ +static +void +ss_mergeforward(const sauchar_t *T, const saidx_t *PA, + saidx_t *first, saidx_t *middle, saidx_t *last, + saidx_t *buf, saidx_t depth) { + saidx_t *a, *b, *c, *bufend; + saidx_t t; + saint_t r; + + bufend = buf + (middle - first) - 1; + ss_blockswap(buf, first, middle - first); + + for(t = *(a = first), b = buf, c = middle;;) { + r = ss_compare(T, PA + *b, PA + *c, depth); + if(r < 0) { + do { + *a++ = *b; + if(bufend <= b) { *bufend = t; return; } + *b++ = *a; + } while(*b < 0); + } else if(r > 0) { + do { + *a++ = *c, *c++ = *a; + if(last <= c) { + while(b < bufend) { *a++ = *b, *b++ = *a; } + *a = *b, *b = t; + return; + } + } while(*c < 0); + } else { + *c = ~*c; + do { + *a++ = *b; + if(bufend <= b) { *bufend = t; return; } + *b++ = *a; + } while(*b < 0); + + do { + *a++ = *c, *c++ = *a; + if(last <= c) { + while(b < bufend) { *a++ = *b, *b++ = *a; } + *a = *b, *b = t; + return; + } + } while(*c < 0); + } + } +} + +/* Merge-backward with internal buffer. */ +static +void +ss_mergebackward(const sauchar_t *T, const saidx_t *PA, + saidx_t *first, saidx_t *middle, saidx_t *last, + saidx_t *buf, saidx_t depth) { + const saidx_t *p1, *p2; + saidx_t *a, *b, *c, *bufend; + saidx_t t; + saint_t r; + saint_t x; + + bufend = buf + (last - middle) - 1; + ss_blockswap(buf, middle, last - middle); + + x = 0; + if(*bufend < 0) { p1 = PA + ~*bufend; x |= 1; } + else { p1 = PA + *bufend; } + if(*(middle - 1) < 0) { p2 = PA + ~*(middle - 1); x |= 2; } + else { p2 = PA + *(middle - 1); } + for(t = *(a = last - 1), b = bufend, c = middle - 1;;) { + r = ss_compare(T, p1, p2, depth); + if(0 < r) { + if(x & 1) { do { *a-- = *b, *b-- = *a; } while(*b < 0); x ^= 1; } + *a-- = *b; + if(b <= buf) { *buf = t; break; } + *b-- = *a; + if(*b < 0) { p1 = PA + ~*b; x |= 1; } + else { p1 = PA + *b; } + } else if(r < 0) { + if(x & 2) { do { *a-- = *c, *c-- = *a; } while(*c < 0); x ^= 2; } + *a-- = *c, *c-- = *a; + if(c < first) { + while(buf < b) { *a-- = *b, *b-- = *a; } + *a = *b, *b = t; + break; + } + if(*c < 0) { p2 = PA + ~*c; x |= 2; } + else { p2 = PA + *c; } + } else { + if(x & 1) { do { *a-- = *b, *b-- = *a; } while(*b < 0); x ^= 1; } + *a-- = ~*b; + if(b <= buf) { *buf = t; break; } + *b-- = *a; + if(x & 2) { do { *a-- = *c, *c-- = *a; } while(*c < 0); x ^= 2; } + *a-- = *c, *c-- = *a; + if(c < first) { + while(buf < b) { *a-- = *b, *b-- = *a; } + *a = *b, *b = t; + break; + } + if(*b < 0) { p1 = PA + ~*b; x |= 1; } + else { p1 = PA + *b; } + if(*c < 0) { p2 = PA + ~*c; x |= 2; } + else { p2 = PA + *c; } + } + } +} + +/* D&C based merge. */ +static +void +ss_swapmerge(const sauchar_t *T, const saidx_t *PA, + saidx_t *first, saidx_t *middle, saidx_t *last, + saidx_t *buf, saidx_t bufsize, saidx_t depth) { +#define STACK_SIZE SS_SMERGE_STACKSIZE +#define GETIDX(a) ((0 <= (a)) ? (a) : (~(a))) +#define MERGE_CHECK(a, b, c)\ + do {\ + if(((c) & 1) ||\ + (((c) & 2) && (ss_compare(T, PA + GETIDX(*((a) - 1)), PA + *(a), depth) == 0))) {\ + *(a) = ~*(a);\ + }\ + if(((c) & 4) && ((ss_compare(T, PA + GETIDX(*((b) - 1)), PA + *(b), depth) == 0))) {\ + *(b) = ~*(b);\ + }\ + } while(0) + struct { saidx_t *a, *b, *c; saint_t d; } stack[STACK_SIZE]; + saidx_t *l, *r, *lm, *rm; + saidx_t m, len, half; + saint_t ssize; + saint_t check, next; + + for(check = 0, ssize = 0;;) { + if((last - middle) <= bufsize) { + if((first < middle) && (middle < last)) { + ss_mergebackward(T, PA, first, middle, last, buf, depth); + } + MERGE_CHECK(first, last, check); + STACK_POP(first, middle, last, check); + continue; + } + + if((middle - first) <= bufsize) { + if(first < middle) { + ss_mergeforward(T, PA, first, middle, last, buf, depth); + } + MERGE_CHECK(first, last, check); + STACK_POP(first, middle, last, check); + continue; + } + + for(m = 0, len = MIN(middle - first, last - middle), half = len >> 1; + 0 < len; + len = half, half >>= 1) { + if(ss_compare(T, PA + GETIDX(*(middle + m + half)), + PA + GETIDX(*(middle - m - half - 1)), depth) < 0) { + m += half + 1; + half -= (len & 1) ^ 1; + } + } + + if(0 < m) { + lm = middle - m, rm = middle + m; + ss_blockswap(lm, middle, m); + l = r = middle, next = 0; + if(rm < last) { + if(*rm < 0) { + *rm = ~*rm; + if(first < lm) { for(; *--l < 0;) { } next |= 4; } + next |= 1; + } else if(first < lm) { + for(; *r < 0; ++r) { } + next |= 2; + } + } + + if((l - first) <= (last - r)) { + STACK_PUSH(r, rm, last, (next & 3) | (check & 4)); + middle = lm, last = l, check = (check & 3) | (next & 4); + } else { + if((next & 2) && (r == middle)) { next ^= 6; } + STACK_PUSH(first, lm, l, (check & 3) | (next & 4)); + first = r, middle = rm, check = (next & 3) | (check & 4); + } + } else { + if(ss_compare(T, PA + GETIDX(*(middle - 1)), PA + *middle, depth) == 0) { + *middle = ~*middle; + } + MERGE_CHECK(first, last, check); + STACK_POP(first, middle, last, check); + } + } +#undef STACK_SIZE +} + +#endif /* SS_BLOCKSIZE != 0 */ + + +/*---------------------------------------------------------------------------*/ + +/*- Function -*/ + +/* Substring sort */ +void +sssort(const sauchar_t *T, const saidx_t *PA, + saidx_t *first, saidx_t *last, + saidx_t *buf, saidx_t bufsize, + saidx_t depth, saidx_t n, saint_t lastsuffix) { + saidx_t *a; +#if SS_BLOCKSIZE != 0 + saidx_t *b, *middle, *curbuf; + saidx_t j, k, curbufsize, limit; +#endif + saidx_t i; + + if(lastsuffix != 0) { ++first; } + +#if SS_BLOCKSIZE == 0 + ss_mintrosort(T, PA, first, last, depth); +#else + if((bufsize < SS_BLOCKSIZE) && + (bufsize < (last - first)) && + (bufsize < (limit = ss_isqrt(last - first)))) { + if(SS_BLOCKSIZE < limit) { limit = SS_BLOCKSIZE; } + buf = middle = last - limit, bufsize = limit; + } else { + middle = last, limit = 0; + } + for(a = first, i = 0; SS_BLOCKSIZE < (middle - a); a += SS_BLOCKSIZE, ++i) { +#if SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE + ss_mintrosort(T, PA, a, a + SS_BLOCKSIZE, depth); +#elif 1 < SS_BLOCKSIZE + ss_insertionsort(T, PA, a, a + SS_BLOCKSIZE, depth); +#endif + curbufsize = last - (a + SS_BLOCKSIZE); + curbuf = a + SS_BLOCKSIZE; + if(curbufsize <= bufsize) { curbufsize = bufsize, curbuf = buf; } + for(b = a, k = SS_BLOCKSIZE, j = i; j & 1; b -= k, k <<= 1, j >>= 1) { + ss_swapmerge(T, PA, b - k, b, b + k, curbuf, curbufsize, depth); + } + } +#if SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE + ss_mintrosort(T, PA, a, middle, depth); +#elif 1 < SS_BLOCKSIZE + ss_insertionsort(T, PA, a, middle, depth); +#endif + for(k = SS_BLOCKSIZE; i != 0; k <<= 1, i >>= 1) { + if(i & 1) { + ss_swapmerge(T, PA, a - k, a, middle, buf, bufsize, depth); + a -= k; + } + } + if(limit != 0) { +#if SS_INSERTIONSORT_THRESHOLD < SS_BLOCKSIZE + ss_mintrosort(T, PA, middle, last, depth); +#elif 1 < SS_BLOCKSIZE + ss_insertionsort(T, PA, middle, last, depth); +#endif + ss_inplacemerge(T, PA, first, middle, last, depth); + } +#endif + + if(lastsuffix != 0) { + /* Insert last type B* suffix. */ + saidx_t PAi[2]; PAi[0] = PA[*(first - 1)], PAi[1] = n - 2; + for(a = first, i = *(first - 1); + (a < last) && ((*a < 0) || (0 < ss_compare(T, &(PAi[0]), PA + *a, depth))); + ++a) { + *(a - 1) = *a; + } + *(a - 1) = i; + } +} diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/lib/trsort.c b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/lib/trsort.c new file mode 100644 index 000000000..6fe3e67ba --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/lib/trsort.c @@ -0,0 +1,586 @@ +/* + * trsort.c for libdivsufsort + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "divsufsort_private.h" + + +/*- Private Functions -*/ + +static const saint_t lg_table[256]= { + -1,0,1,1,2,2,2,2,3,3,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4, + 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5, + 6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6, + 6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7 +}; + +static INLINE +saint_t +tr_ilg(saidx_t n) { +#if defined(BUILD_DIVSUFSORT64) + return (n >> 32) ? + ((n >> 48) ? + ((n >> 56) ? + 56 + lg_table[(n >> 56) & 0xff] : + 48 + lg_table[(n >> 48) & 0xff]) : + ((n >> 40) ? + 40 + lg_table[(n >> 40) & 0xff] : + 32 + lg_table[(n >> 32) & 0xff])) : + ((n & 0xffff0000) ? + ((n & 0xff000000) ? + 24 + lg_table[(n >> 24) & 0xff] : + 16 + lg_table[(n >> 16) & 0xff]) : + ((n & 0x0000ff00) ? + 8 + lg_table[(n >> 8) & 0xff] : + 0 + lg_table[(n >> 0) & 0xff])); +#else + return (n & 0xffff0000) ? + ((n & 0xff000000) ? + 24 + lg_table[(n >> 24) & 0xff] : + 16 + lg_table[(n >> 16) & 0xff]) : + ((n & 0x0000ff00) ? + 8 + lg_table[(n >> 8) & 0xff] : + 0 + lg_table[(n >> 0) & 0xff]); +#endif +} + + +/*---------------------------------------------------------------------------*/ + +/* Simple insertionsort for small size groups. */ +static +void +tr_insertionsort(const saidx_t *ISAd, saidx_t *first, saidx_t *last) { + saidx_t *a, *b; + saidx_t t, r; + + for(a = first + 1; a < last; ++a) { + for(t = *a, b = a - 1; 0 > (r = ISAd[t] - ISAd[*b]);) { + do { *(b + 1) = *b; } while((first <= --b) && (*b < 0)); + if(b < first) { break; } + } + if(r == 0) { *b = ~*b; } + *(b + 1) = t; + } +} + + +/*---------------------------------------------------------------------------*/ + +static INLINE +void +tr_fixdown(const saidx_t *ISAd, saidx_t *SA, saidx_t i, saidx_t size) { + saidx_t j, k; + saidx_t v; + saidx_t c, d, e; + + for(v = SA[i], c = ISAd[v]; (j = 2 * i + 1) < size; SA[i] = SA[k], i = k) { + d = ISAd[SA[k = j++]]; + if(d < (e = ISAd[SA[j]])) { k = j; d = e; } + if(d <= c) { break; } + } + SA[i] = v; +} + +/* Simple top-down heapsort. */ +static +void +tr_heapsort(const saidx_t *ISAd, saidx_t *SA, saidx_t size) { + saidx_t i, m; + saidx_t t; + + m = size; + if((size % 2) == 0) { + m--; + if(ISAd[SA[m / 2]] < ISAd[SA[m]]) { SWAP(SA[m], SA[m / 2]); } + } + + for(i = m / 2 - 1; 0 <= i; --i) { tr_fixdown(ISAd, SA, i, m); } + if((size % 2) == 0) { SWAP(SA[0], SA[m]); tr_fixdown(ISAd, SA, 0, m); } + for(i = m - 1; 0 < i; --i) { + t = SA[0], SA[0] = SA[i]; + tr_fixdown(ISAd, SA, 0, i); + SA[i] = t; + } +} + + +/*---------------------------------------------------------------------------*/ + +/* Returns the median of three elements. */ +static INLINE +saidx_t * +tr_median3(const saidx_t *ISAd, saidx_t *v1, saidx_t *v2, saidx_t *v3) { + saidx_t *t; + if(ISAd[*v1] > ISAd[*v2]) { SWAP(v1, v2); } + if(ISAd[*v2] > ISAd[*v3]) { + if(ISAd[*v1] > ISAd[*v3]) { return v1; } + else { return v3; } + } + return v2; +} + +/* Returns the median of five elements. */ +static INLINE +saidx_t * +tr_median5(const saidx_t *ISAd, + saidx_t *v1, saidx_t *v2, saidx_t *v3, saidx_t *v4, saidx_t *v5) { + saidx_t *t; + if(ISAd[*v2] > ISAd[*v3]) { SWAP(v2, v3); } + if(ISAd[*v4] > ISAd[*v5]) { SWAP(v4, v5); } + if(ISAd[*v2] > ISAd[*v4]) { SWAP(v2, v4); SWAP(v3, v5); } + if(ISAd[*v1] > ISAd[*v3]) { SWAP(v1, v3); } + if(ISAd[*v1] > ISAd[*v4]) { SWAP(v1, v4); SWAP(v3, v5); } + if(ISAd[*v3] > ISAd[*v4]) { return v4; } + return v3; +} + +/* Returns the pivot element. */ +static INLINE +saidx_t * +tr_pivot(const saidx_t *ISAd, saidx_t *first, saidx_t *last) { + saidx_t *middle; + saidx_t t; + + t = last - first; + middle = first + t / 2; + + if(t <= 512) { + if(t <= 32) { + return tr_median3(ISAd, first, middle, last - 1); + } else { + t >>= 2; + return tr_median5(ISAd, first, first + t, middle, last - 1 - t, last - 1); + } + } + t >>= 3; + first = tr_median3(ISAd, first, first + t, first + (t << 1)); + middle = tr_median3(ISAd, middle - t, middle, middle + t); + last = tr_median3(ISAd, last - 1 - (t << 1), last - 1 - t, last - 1); + return tr_median3(ISAd, first, middle, last); +} + + +/*---------------------------------------------------------------------------*/ + +typedef struct _trbudget_t trbudget_t; +struct _trbudget_t { + saidx_t chance; + saidx_t remain; + saidx_t incval; + saidx_t count; +}; + +static INLINE +void +trbudget_init(trbudget_t *budget, saidx_t chance, saidx_t incval) { + budget->chance = chance; + budget->remain = budget->incval = incval; +} + +static INLINE +saint_t +trbudget_check(trbudget_t *budget, saidx_t size) { + if(size <= budget->remain) { budget->remain -= size; return 1; } + if(budget->chance == 0) { budget->count += size; return 0; } + budget->remain += budget->incval - size; + budget->chance -= 1; + return 1; +} + + +/*---------------------------------------------------------------------------*/ + +static INLINE +void +tr_partition(const saidx_t *ISAd, + saidx_t *first, saidx_t *middle, saidx_t *last, + saidx_t **pa, saidx_t **pb, saidx_t v) { + saidx_t *a, *b, *c, *d, *e, *f; + saidx_t t, s; + saidx_t x = 0; + + for(b = middle - 1; (++b < last) && ((x = ISAd[*b]) == v);) { } + if(((a = b) < last) && (x < v)) { + for(; (++b < last) && ((x = ISAd[*b]) <= v);) { + if(x == v) { SWAP(*b, *a); ++a; } + } + } + for(c = last; (b < --c) && ((x = ISAd[*c]) == v);) { } + if((b < (d = c)) && (x > v)) { + for(; (b < --c) && ((x = ISAd[*c]) >= v);) { + if(x == v) { SWAP(*c, *d); --d; } + } + } + for(; b < c;) { + SWAP(*b, *c); + for(; (++b < c) && ((x = ISAd[*b]) <= v);) { + if(x == v) { SWAP(*b, *a); ++a; } + } + for(; (b < --c) && ((x = ISAd[*c]) >= v);) { + if(x == v) { SWAP(*c, *d); --d; } + } + } + + if(a <= d) { + c = b - 1; + if((s = a - first) > (t = b - a)) { s = t; } + for(e = first, f = b - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); } + if((s = d - c) > (t = last - d - 1)) { s = t; } + for(e = b, f = last - s; 0 < s; --s, ++e, ++f) { SWAP(*e, *f); } + first += (b - a), last -= (d - c); + } + *pa = first, *pb = last; +} + +static +void +tr_copy(saidx_t *ISA, const saidx_t *SA, + saidx_t *first, saidx_t *a, saidx_t *b, saidx_t *last, + saidx_t depth) { + /* sort suffixes of middle partition + by using sorted order of suffixes of left and right partition. */ + saidx_t *c, *d, *e; + saidx_t s, v; + + v = b - SA - 1; + for(c = first, d = a - 1; c <= d; ++c) { + if((0 <= (s = *c - depth)) && (ISA[s] == v)) { + *++d = s; + ISA[s] = d - SA; + } + } + for(c = last - 1, e = d + 1, d = b; e < d; --c) { + if((0 <= (s = *c - depth)) && (ISA[s] == v)) { + *--d = s; + ISA[s] = d - SA; + } + } +} + +static +void +tr_partialcopy(saidx_t *ISA, const saidx_t *SA, + saidx_t *first, saidx_t *a, saidx_t *b, saidx_t *last, + saidx_t depth) { + saidx_t *c, *d, *e; + saidx_t s, v; + saidx_t rank, lastrank, newrank = -1; + + v = b - SA - 1; + lastrank = -1; + for(c = first, d = a - 1; c <= d; ++c) { + if((0 <= (s = *c - depth)) && (ISA[s] == v)) { + *++d = s; + rank = ISA[s + depth]; + if(lastrank != rank) { lastrank = rank; newrank = d - SA; } + ISA[s] = newrank; + } + } + + lastrank = -1; + for(e = d; first <= e; --e) { + rank = ISA[*e]; + if(lastrank != rank) { lastrank = rank; newrank = e - SA; } + if(newrank != rank) { ISA[*e] = newrank; } + } + + lastrank = -1; + for(c = last - 1, e = d + 1, d = b; e < d; --c) { + if((0 <= (s = *c - depth)) && (ISA[s] == v)) { + *--d = s; + rank = ISA[s + depth]; + if(lastrank != rank) { lastrank = rank; newrank = d - SA; } + ISA[s] = newrank; + } + } +} + +static +void +tr_introsort(saidx_t *ISA, const saidx_t *ISAd, + saidx_t *SA, saidx_t *first, saidx_t *last, + trbudget_t *budget) { +#define STACK_SIZE TR_STACKSIZE + struct { const saidx_t *a; saidx_t *b, *c; saint_t d, e; }stack[STACK_SIZE]; + saidx_t *a, *b, *c; + saidx_t t; + saidx_t v, x = 0; + saidx_t incr = ISAd - ISA; + saint_t limit, next; + saint_t ssize, trlink = -1; + + for(ssize = 0, limit = tr_ilg(last - first);;) { + + if(limit < 0) { + if(limit == -1) { + /* tandem repeat partition */ + tr_partition(ISAd - incr, first, first, last, &a, &b, last - SA - 1); + + /* update ranks */ + if(a < last) { + for(c = first, v = a - SA - 1; c < a; ++c) { ISA[*c] = v; } + } + if(b < last) { + for(c = a, v = b - SA - 1; c < b; ++c) { ISA[*c] = v; } + } + + /* push */ + if(1 < (b - a)) { + STACK_PUSH5(NULL, a, b, 0, 0); + STACK_PUSH5(ISAd - incr, first, last, -2, trlink); + trlink = ssize - 2; + } + if((a - first) <= (last - b)) { + if(1 < (a - first)) { + STACK_PUSH5(ISAd, b, last, tr_ilg(last - b), trlink); + last = a, limit = tr_ilg(a - first); + } else if(1 < (last - b)) { + first = b, limit = tr_ilg(last - b); + } else { + STACK_POP5(ISAd, first, last, limit, trlink); + } + } else { + if(1 < (last - b)) { + STACK_PUSH5(ISAd, first, a, tr_ilg(a - first), trlink); + first = b, limit = tr_ilg(last - b); + } else if(1 < (a - first)) { + last = a, limit = tr_ilg(a - first); + } else { + STACK_POP5(ISAd, first, last, limit, trlink); + } + } + } else if(limit == -2) { + /* tandem repeat copy */ + a = stack[--ssize].b, b = stack[ssize].c; + if(stack[ssize].d == 0) { + tr_copy(ISA, SA, first, a, b, last, ISAd - ISA); + } else { + if(0 <= trlink) { stack[trlink].d = -1; } + tr_partialcopy(ISA, SA, first, a, b, last, ISAd - ISA); + } + STACK_POP5(ISAd, first, last, limit, trlink); + } else { + /* sorted partition */ + if(0 <= *first) { + a = first; + do { ISA[*a] = a - SA; } while((++a < last) && (0 <= *a)); + first = a; + } + if(first < last) { + a = first; do { *a = ~*a; } while(*++a < 0); + next = (ISA[*a] != ISAd[*a]) ? tr_ilg(a - first + 1) : -1; + if(++a < last) { for(b = first, v = a - SA - 1; b < a; ++b) { ISA[*b] = v; } } + + /* push */ + if(trbudget_check(budget, a - first)) { + if((a - first) <= (last - a)) { + STACK_PUSH5(ISAd, a, last, -3, trlink); + ISAd += incr, last = a, limit = next; + } else { + if(1 < (last - a)) { + STACK_PUSH5(ISAd + incr, first, a, next, trlink); + first = a, limit = -3; + } else { + ISAd += incr, last = a, limit = next; + } + } + } else { + if(0 <= trlink) { stack[trlink].d = -1; } + if(1 < (last - a)) { + first = a, limit = -3; + } else { + STACK_POP5(ISAd, first, last, limit, trlink); + } + } + } else { + STACK_POP5(ISAd, first, last, limit, trlink); + } + } + continue; + } + + if((last - first) <= TR_INSERTIONSORT_THRESHOLD) { + tr_insertionsort(ISAd, first, last); + limit = -3; + continue; + } + + if(limit-- == 0) { + tr_heapsort(ISAd, first, last - first); + for(a = last - 1; first < a; a = b) { + for(x = ISAd[*a], b = a - 1; (first <= b) && (ISAd[*b] == x); --b) { *b = ~*b; } + } + limit = -3; + continue; + } + + /* choose pivot */ + a = tr_pivot(ISAd, first, last); + SWAP(*first, *a); + v = ISAd[*first]; + + /* partition */ + tr_partition(ISAd, first, first + 1, last, &a, &b, v); + if((last - first) != (b - a)) { + next = (ISA[*a] != v) ? tr_ilg(b - a) : -1; + + /* update ranks */ + for(c = first, v = a - SA - 1; c < a; ++c) { ISA[*c] = v; } + if(b < last) { for(c = a, v = b - SA - 1; c < b; ++c) { ISA[*c] = v; } } + + /* push */ + if((1 < (b - a)) && (trbudget_check(budget, b - a))) { + if((a - first) <= (last - b)) { + if((last - b) <= (b - a)) { + if(1 < (a - first)) { + STACK_PUSH5(ISAd + incr, a, b, next, trlink); + STACK_PUSH5(ISAd, b, last, limit, trlink); + last = a; + } else if(1 < (last - b)) { + STACK_PUSH5(ISAd + incr, a, b, next, trlink); + first = b; + } else { + ISAd += incr, first = a, last = b, limit = next; + } + } else if((a - first) <= (b - a)) { + if(1 < (a - first)) { + STACK_PUSH5(ISAd, b, last, limit, trlink); + STACK_PUSH5(ISAd + incr, a, b, next, trlink); + last = a; + } else { + STACK_PUSH5(ISAd, b, last, limit, trlink); + ISAd += incr, first = a, last = b, limit = next; + } + } else { + STACK_PUSH5(ISAd, b, last, limit, trlink); + STACK_PUSH5(ISAd, first, a, limit, trlink); + ISAd += incr, first = a, last = b, limit = next; + } + } else { + if((a - first) <= (b - a)) { + if(1 < (last - b)) { + STACK_PUSH5(ISAd + incr, a, b, next, trlink); + STACK_PUSH5(ISAd, first, a, limit, trlink); + first = b; + } else if(1 < (a - first)) { + STACK_PUSH5(ISAd + incr, a, b, next, trlink); + last = a; + } else { + ISAd += incr, first = a, last = b, limit = next; + } + } else if((last - b) <= (b - a)) { + if(1 < (last - b)) { + STACK_PUSH5(ISAd, first, a, limit, trlink); + STACK_PUSH5(ISAd + incr, a, b, next, trlink); + first = b; + } else { + STACK_PUSH5(ISAd, first, a, limit, trlink); + ISAd += incr, first = a, last = b, limit = next; + } + } else { + STACK_PUSH5(ISAd, first, a, limit, trlink); + STACK_PUSH5(ISAd, b, last, limit, trlink); + ISAd += incr, first = a, last = b, limit = next; + } + } + } else { + if((1 < (b - a)) && (0 <= trlink)) { stack[trlink].d = -1; } + if((a - first) <= (last - b)) { + if(1 < (a - first)) { + STACK_PUSH5(ISAd, b, last, limit, trlink); + last = a; + } else if(1 < (last - b)) { + first = b; + } else { + STACK_POP5(ISAd, first, last, limit, trlink); + } + } else { + if(1 < (last - b)) { + STACK_PUSH5(ISAd, first, a, limit, trlink); + first = b; + } else if(1 < (a - first)) { + last = a; + } else { + STACK_POP5(ISAd, first, last, limit, trlink); + } + } + } + } else { + if(trbudget_check(budget, last - first)) { + limit = tr_ilg(last - first), ISAd += incr; + } else { + if(0 <= trlink) { stack[trlink].d = -1; } + STACK_POP5(ISAd, first, last, limit, trlink); + } + } + } +#undef STACK_SIZE +} + + + +/*---------------------------------------------------------------------------*/ + +/*- Function -*/ + +/* Tandem repeat sort */ +void +trsort(saidx_t *ISA, saidx_t *SA, saidx_t n, saidx_t depth) { + saidx_t *ISAd; + saidx_t *first, *last; + trbudget_t budget; + saidx_t t, skip, unsorted; + + trbudget_init(&budget, tr_ilg(n) * 2 / 3, n); +/* trbudget_init(&budget, tr_ilg(n) * 3 / 4, n); */ + for(ISAd = ISA + depth; -n < *SA; ISAd += ISAd - ISA) { + first = SA; + skip = 0; + unsorted = 0; + do { + if((t = *first) < 0) { first -= t; skip += t; } + else { + if(skip != 0) { *(first + skip) = skip; skip = 0; } + last = SA + ISA[t] + 1; + if(1 < (last - first)) { + budget.count = 0; + tr_introsort(ISA, ISAd, SA, first, last, &budget); + if(budget.count != 0) { unsorted += budget.count; } + else { skip = first - last; } + } else if((last - first) == 1) { + skip = -1; + } + first = last; + } + } while(first < (SA + n)); + if(skip != 0) { *(first + skip) = skip; } + if(unsorted == 0) { break; } + } +} diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/lib/utils.c b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/lib/utils.c new file mode 100644 index 000000000..90fb23efa --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/lib/utils.c @@ -0,0 +1,381 @@ +/* + * utils.c for libdivsufsort + * Copyright (c) 2003-2008 Yuta Mori All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person + * obtaining a copy of this software and associated documentation + * files (the "Software"), to deal in the Software without + * restriction, including without limitation the rights to use, + * copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following + * conditions: + * + * The above copyright notice and this permission notice shall be + * included in all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES + * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT + * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, + * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR + * OTHER DEALINGS IN THE SOFTWARE. + */ + +#include "divsufsort_private.h" + + +/*- Private Function -*/ + +/* Binary search for inverse bwt. */ +static +saidx_t +binarysearch_lower(const saidx_t *A, saidx_t size, saidx_t value) { + saidx_t half, i; + for(i = 0, half = size >> 1; + 0 < size; + size = half, half >>= 1) { + if(A[i + half] < value) { + i += half + 1; + half -= (size & 1) ^ 1; + } + } + return i; +} + + +/*- Functions -*/ + +/* Burrows-Wheeler transform. */ +saint_t +bw_transform(const sauchar_t *T, sauchar_t *U, saidx_t *SA, + saidx_t n, saidx_t *idx) { + saidx_t *A, i, j, p, t; + saint_t c; + + /* Check arguments. */ + if((T == NULL) || (U == NULL) || (n < 0) || (idx == NULL)) { return -1; } + if(n <= 1) { + if(n == 1) { U[0] = T[0]; } + *idx = n; + return 0; + } + + if((A = SA) == NULL) { + i = divbwt(T, U, NULL, n); + if(0 <= i) { *idx = i; i = 0; } + return (saint_t)i; + } + + /* BW transform. */ + if(T == U) { + t = n; + for(i = 0, j = 0; i < n; ++i) { + p = t - 1; + t = A[i]; + if(0 <= p) { + c = T[j]; + U[j] = (j <= p) ? T[p] : (sauchar_t)A[p]; + A[j] = c; + j++; + } else { + *idx = i; + } + } + p = t - 1; + if(0 <= p) { + c = T[j]; + U[j] = (j <= p) ? T[p] : (sauchar_t)A[p]; + A[j] = c; + } else { + *idx = i; + } + } else { + U[0] = T[n - 1]; + for(i = 0; A[i] != 0; ++i) { U[i + 1] = T[A[i] - 1]; } + *idx = i + 1; + for(++i; i < n; ++i) { U[i] = T[A[i] - 1]; } + } + + if(SA == NULL) { + /* Deallocate memory. */ + free(A); + } + + return 0; +} + +/* Inverse Burrows-Wheeler transform. */ +saint_t +inverse_bw_transform(const sauchar_t *T, sauchar_t *U, saidx_t *A, + saidx_t n, saidx_t idx) { + saidx_t C[ALPHABET_SIZE]; + sauchar_t D[ALPHABET_SIZE]; + saidx_t *B; + saidx_t i, p; + saint_t c, d; + + /* Check arguments. */ + if((T == NULL) || (U == NULL) || (n < 0) || (idx < 0) || + (n < idx) || ((0 < n) && (idx == 0))) { + return -1; + } + if(n <= 1) { return 0; } + + if((B = A) == NULL) { + /* Allocate n*sizeof(saidx_t) bytes of memory. */ + if((B = (saidx_t *)malloc((size_t)n * sizeof(saidx_t))) == NULL) { return -2; } + } + + /* Inverse BW transform. */ + for(c = 0; c < ALPHABET_SIZE; ++c) { C[c] = 0; } + for(i = 0; i < n; ++i) { ++C[T[i]]; } + for(c = 0, d = 0, i = 0; c < ALPHABET_SIZE; ++c) { + p = C[c]; + if(0 < p) { + C[c] = i; + D[d++] = (sauchar_t)c; + i += p; + } + } + for(i = 0; i < idx; ++i) { B[C[T[i]]++] = i; } + for( ; i < n; ++i) { B[C[T[i]]++] = i + 1; } + for(c = 0; c < d; ++c) { C[c] = C[D[c]]; } + for(i = 0, p = idx; i < n; ++i) { + U[i] = D[binarysearch_lower(C, d, p)]; + p = B[p - 1]; + } + + if(A == NULL) { + /* Deallocate memory. */ + free(B); + } + + return 0; +} + +/* Checks the suffix array SA of the string T. */ +saint_t +sufcheck(const sauchar_t *T, const saidx_t *SA, + saidx_t n, saint_t verbose) { + saidx_t C[ALPHABET_SIZE]; + saidx_t i, p, q, t; + saint_t c; + + if(verbose) { fprintf(stderr, "sufcheck: "); } + + /* Check arguments. */ + if((T == NULL) || (SA == NULL) || (n < 0)) { + if(verbose) { fprintf(stderr, "Invalid arguments.\n"); } + return -1; + } + if(n == 0) { + if(verbose) { fprintf(stderr, "Done.\n"); } + return 0; + } + + /* check range: [0..n-1] */ + for(i = 0; i < n; ++i) { + if((SA[i] < 0) || (n <= SA[i])) { + if(verbose) { + fprintf(stderr, "Out of the range [0,%" PRIdSAIDX_T "].\n" + " SA[%" PRIdSAIDX_T "]=%" PRIdSAIDX_T "\n", + n - 1, i, SA[i]); + } + return -2; + } + } + + /* check first characters. */ + for(i = 1; i < n; ++i) { + if(T[SA[i - 1]] > T[SA[i]]) { + if(verbose) { + fprintf(stderr, "Suffixes in wrong order.\n" + " T[SA[%" PRIdSAIDX_T "]=%" PRIdSAIDX_T "]=%d" + " > T[SA[%" PRIdSAIDX_T "]=%" PRIdSAIDX_T "]=%d\n", + i - 1, SA[i - 1], T[SA[i - 1]], i, SA[i], T[SA[i]]); + } + return -3; + } + } + + /* check suffixes. */ + for(i = 0; i < ALPHABET_SIZE; ++i) { C[i] = 0; } + for(i = 0; i < n; ++i) { ++C[T[i]]; } + for(i = 0, p = 0; i < ALPHABET_SIZE; ++i) { + t = C[i]; + C[i] = p; + p += t; + } + + q = C[T[n - 1]]; + C[T[n - 1]] += 1; + for(i = 0; i < n; ++i) { + p = SA[i]; + if(0 < p) { + c = T[--p]; + t = C[c]; + } else { + c = T[p = n - 1]; + t = q; + } + if((t < 0) || (p != SA[t])) { + if(verbose) { + fprintf(stderr, "Suffix in wrong position.\n" + " SA[%" PRIdSAIDX_T "]=%" PRIdSAIDX_T " or\n" + " SA[%" PRIdSAIDX_T "]=%" PRIdSAIDX_T "\n", + t, (0 <= t) ? SA[t] : -1, i, SA[i]); + } + return -4; + } + if(t != q) { + ++C[c]; + if((n <= C[c]) || (T[SA[C[c]]] != c)) { C[c] = -1; } + } + } + + if(1 <= verbose) { fprintf(stderr, "Done.\n"); } + return 0; +} + + +static +int +_compare(const sauchar_t *T, saidx_t Tsize, + const sauchar_t *P, saidx_t Psize, + saidx_t suf, saidx_t *match) { + saidx_t i, j; + saint_t r; + for(i = suf + *match, j = *match, r = 0; + (i < Tsize) && (j < Psize) && ((r = T[i] - P[j]) == 0); ++i, ++j) { } + *match = j; + return (r == 0) ? -(j != Psize) : r; +} + +/* Search for the pattern P in the string T. */ +saidx_t +sa_search(const sauchar_t *T, saidx_t Tsize, + const sauchar_t *P, saidx_t Psize, + const saidx_t *SA, saidx_t SAsize, + saidx_t *idx) { + saidx_t size, lsize, rsize, half; + saidx_t match, lmatch, rmatch; + saidx_t llmatch, lrmatch, rlmatch, rrmatch; + saidx_t i, j, k; + saint_t r; + + if(idx != NULL) { *idx = -1; } + if((T == NULL) || (P == NULL) || (SA == NULL) || + (Tsize < 0) || (Psize < 0) || (SAsize < 0)) { return -1; } + if((Tsize == 0) || (SAsize == 0)) { return 0; } + if(Psize == 0) { if(idx != NULL) { *idx = 0; } return SAsize; } + + for(i = j = k = 0, lmatch = rmatch = 0, size = SAsize, half = size >> 1; + 0 < size; + size = half, half >>= 1) { + match = MIN(lmatch, rmatch); + r = _compare(T, Tsize, P, Psize, SA[i + half], &match); + if(r < 0) { + i += half + 1; + half -= (size & 1) ^ 1; + lmatch = match; + } else if(r > 0) { + rmatch = match; + } else { + lsize = half, j = i, rsize = size - half - 1, k = i + half + 1; + + /* left part */ + for(llmatch = lmatch, lrmatch = match, half = lsize >> 1; + 0 < lsize; + lsize = half, half >>= 1) { + lmatch = MIN(llmatch, lrmatch); + r = _compare(T, Tsize, P, Psize, SA[j + half], &lmatch); + if(r < 0) { + j += half + 1; + half -= (lsize & 1) ^ 1; + llmatch = lmatch; + } else { + lrmatch = lmatch; + } + } + + /* right part */ + for(rlmatch = match, rrmatch = rmatch, half = rsize >> 1; + 0 < rsize; + rsize = half, half >>= 1) { + rmatch = MIN(rlmatch, rrmatch); + r = _compare(T, Tsize, P, Psize, SA[k + half], &rmatch); + if(r <= 0) { + k += half + 1; + half -= (rsize & 1) ^ 1; + rlmatch = rmatch; + } else { + rrmatch = rmatch; + } + } + + break; + } + } + + if(idx != NULL) { *idx = (0 < (k - j)) ? j : i; } + return k - j; +} + +/* Search for the character c in the string T. */ +saidx_t +sa_simplesearch(const sauchar_t *T, saidx_t Tsize, + const saidx_t *SA, saidx_t SAsize, + saint_t c, saidx_t *idx) { + saidx_t size, lsize, rsize, half; + saidx_t i, j, k, p; + saint_t r; + + if(idx != NULL) { *idx = -1; } + if((T == NULL) || (SA == NULL) || (Tsize < 0) || (SAsize < 0)) { return -1; } + if((Tsize == 0) || (SAsize == 0)) { return 0; } + + for(i = j = k = 0, size = SAsize, half = size >> 1; + 0 < size; + size = half, half >>= 1) { + p = SA[i + half]; + r = (p < Tsize) ? T[p] - c : -1; + if(r < 0) { + i += half + 1; + half -= (size & 1) ^ 1; + } else if(r == 0) { + lsize = half, j = i, rsize = size - half - 1, k = i + half + 1; + + /* left part */ + for(half = lsize >> 1; + 0 < lsize; + lsize = half, half >>= 1) { + p = SA[j + half]; + r = (p < Tsize) ? T[p] - c : -1; + if(r < 0) { + j += half + 1; + half -= (lsize & 1) ^ 1; + } + } + + /* right part */ + for(half = rsize >> 1; + 0 < rsize; + rsize = half, half >>= 1) { + p = SA[k + half]; + r = (p < Tsize) ? T[p] - c : -1; + if(r <= 0) { + k += half + 1; + half -= (rsize & 1) ^ 1; + } + } + + break; + } + } + + if(idx != NULL) { *idx = (0 < (k - j)) ? j : i; } + return k - j; +} diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/pkgconfig/CMakeLists.txt b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/pkgconfig/CMakeLists.txt new file mode 100644 index 000000000..ee7063c98 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/pkgconfig/CMakeLists.txt @@ -0,0 +1,9 @@ +## generate libdivsufsort.pc ## +set(W64BIT "") +configure_file("${CMAKE_CURRENT_SOURCE_DIR}/libdivsufsort.pc.cmake" "${CMAKE_CURRENT_BINARY_DIR}/libdivsufsort.pc" @ONLY) +install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libdivsufsort.pc" DESTINATION ${CMAKE_INSTALL_PKGCONFIGDIR}) +if(BUILD_DIVSUFSORT64) + set(W64BIT "64") + configure_file("${CMAKE_CURRENT_SOURCE_DIR}/libdivsufsort.pc.cmake" "${CMAKE_CURRENT_BINARY_DIR}/libdivsufsort64.pc" @ONLY) + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libdivsufsort64.pc" DESTINATION ${CMAKE_INSTALL_PKGCONFIGDIR}) +endif(BUILD_DIVSUFSORT64) diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/pkgconfig/libdivsufsort.pc.cmake b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/pkgconfig/libdivsufsort.pc.cmake new file mode 100644 index 000000000..6419d1ea8 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/libdivsufsort/pkgconfig/libdivsufsort.pc.cmake @@ -0,0 +1,11 @@ +prefix=@CMAKE_INSTALL_PREFIX@ +exec_prefix=${prefix} +libdir=@CMAKE_INSTALL_LIBDIR@ +includedir=@CMAKE_INSTALL_INCLUDEDIR@ + +Name: @PROJECT_NAME@@W64BIT@ +Description: @PROJECT_DESCRIPTION@ +Version: @PROJECT_VERSION_FULL@ +URL: @PROJECT_URL@ +Libs: -L${libdir} -ldivsufsort@W64BIT@ +Cflags: -I${includedir} diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/read_dist.h b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/read_dist.h new file mode 100644 index 000000000..63d3b97ff --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/read_dist.h @@ -0,0 +1,50 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + Author: zip753@gmail.com (Ivan Nikulin) + + Distributed under MIT license. + See file LICENSE for detail or copy at https://opensource.org/licenses/MIT +*/ + +/* API for reading distances from *.dist file. + The format of *.dist file is as follows: for each backward reference there is + a position-distance pair, also a copy length may be specified. Copy length is + prefixed with flag byte 0, position-distance pair is prefixed with flag + byte 1. Each number is a 32-bit integer. Copy length always comes before + position-distance pair. Standalone copy length is allowed, in this case it is + ignored. */ + +#ifndef BROTLI_RESEARCH_READ_DIST_H_ +#define BROTLI_RESEARCH_READ_DIST_H_ + +#include <cstdio> +#include <cstdlib> /* exit, EXIT_FAILURE */ + +#if !defined(CHECK) +#define CHECK(X) if (!(X)) exit(EXIT_FAILURE); +#endif + +/* Reads backwards reference from .dist file. Sets all missing fields to -1. + Returns false when EOF is met or input is corrupt. */ +bool ReadBackwardReference(FILE* fin, int* copy, int* pos, int* dist) { + int c = getc(fin); + if (c == EOF) return false; + if (c == 0) { + CHECK(fread(copy, sizeof(int), 1, fin) == 1); + if ((c = getc(fin)) != 1) { + ungetc(c, fin); + *pos = *dist = -1; + } else { + CHECK(fread(pos, sizeof(int), 1, fin) == 1); + CHECK(fread(dist, sizeof(int), 1, fin) == 1); + } + } else if (c != 1) { + return false; + } else { + CHECK(fread(pos, sizeof(int), 1, fin) == 1); + CHECK(fread(dist, sizeof(int), 1, fin) == 1); + *copy = -1; + } + return true; +} + +#endif /* BROTLI_RESEARCH_READ_DIST_H_ */ diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/sieve.cc b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/sieve.cc new file mode 100755 index 000000000..4d147e112 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/sieve.cc @@ -0,0 +1,259 @@ +#include "./sieve.h" + +/* Pointer to position in (combined corpus) text. */ +typedef uint32_t TextIdx; + +/* Index of sample / generation. */ +typedef uint16_t SampleIdx; + +typedef struct Slot { + TextIdx next; + TextIdx offset; + SampleIdx presence; + SampleIdx mark; +} Slot; + +static const TextIdx kNowhere = static_cast<TextIdx>(-1); + +static TextIdx dryRun(TextIdx sliceLen, Slot* map, TextIdx* shortcut, + TextIdx end, TextIdx middle, SampleIdx minPresence, SampleIdx iteration) { + TextIdx from = kNowhere; + TextIdx to = kNowhere; + TextIdx result = 0; + SampleIdx targetPresence = minPresence; + for (TextIdx i = 0; i < end; ++i) { + if (i == middle) { + targetPresence++; + } + Slot& item = map[shortcut[i]]; + if (item.mark != iteration) { + item.mark = iteration; + if (item.presence >= targetPresence) { + if ((to == kNowhere) || (to < i)) { + if (from != kNowhere) { + result += to - from; + } + from = i; + } + to = i + sliceLen; + } + } + } + if (from != kNowhere) { + result += to - from; + } + return result; +} + +static std::string createDictionary(const uint8_t* data, TextIdx sliceLen, + Slot* map, TextIdx* shortcut, TextIdx end, TextIdx middle, + SampleIdx minPresence, SampleIdx iteration) { + std::string output; + TextIdx from = kNowhere; + TextIdx to = kNowhere; + SampleIdx targetPresence = minPresence; + for (TextIdx i = 0; i < end; ++i) { + if (i == middle) { + targetPresence++; + } + Slot& item = map[shortcut[i]]; + if (item.mark != iteration) { + item.mark = iteration; + if (item.presence >= targetPresence) { + if ((to == kNowhere) || (to < i)) { + if (from != kNowhere) { + output.insert(output.end(), &data[from], &data[to]); + } + from = i; + } + to = i + sliceLen; + } + } + } + if (from != kNowhere) { + output.insert(output.end(), &data[from], &data[to]); + } + return output; +} + +std::string sieve_generate(size_t dictionary_size_limit, size_t slice_len, + const std::vector<size_t>& sample_sizes, const uint8_t* sample_data) { + /* Parameters aliasing */ + TextIdx targetSize = static_cast<TextIdx>(dictionary_size_limit); + if (targetSize != dictionary_size_limit) { + fprintf(stderr, "dictionary_size_limit is too large\n"); + return ""; + } + TextIdx sliceLen = static_cast<TextIdx>(slice_len); + if (sliceLen != slice_len) { + fprintf(stderr, "slice_len is too large\n"); + return ""; + } + if (sliceLen < 1) { + fprintf(stderr, "slice_len is too small\n"); + return ""; + } + SampleIdx numSamples = static_cast<SampleIdx>(sample_sizes.size()); + if ((numSamples != sample_sizes.size()) || (numSamples * 2 < numSamples)) { + fprintf(stderr, "too many samples\n"); + return ""; + } + const uint8_t* data = sample_data; + + TextIdx total = 0; + std::vector<TextIdx> offsets; + for (SampleIdx i = 0; i < numSamples; ++i) { + TextIdx delta = static_cast<TextIdx>(sample_sizes[i]); + if (delta != sample_sizes[i]) { + fprintf(stderr, "sample is too large\n"); + return ""; + } + if (delta == 0) { + fprintf(stderr, "empty samples are prohibited\n"); + return ""; + } + if (total + delta <= total) { + fprintf(stderr, "corpus is too large\n"); + return ""; + } + total += delta; + offsets.push_back(total); + } + + if (total * 2 < total) { + fprintf(stderr, "corpus is too large\n"); + return ""; + } + + if (total < sliceLen) { + fprintf(stderr, "slice_len is larger than corpus size\n"); + return ""; + } + + /***************************************************************************** + * Build coverage map. + ****************************************************************************/ + std::vector<Slot> map; + std::vector<TextIdx> shortcut; + map.push_back({0, 0, 0, 0}); + TextIdx end = total - sliceLen; + TextIdx hashLen = 11; + while (hashLen < 29 && ((1u << hashLen) < end)) { + hashLen += 3; + } + hashLen -= 3; + TextIdx hashMask = (1u << hashLen) - 1u; + std::vector<TextIdx> hashHead(1 << hashLen); + TextIdx hashSlot = 1; + SampleIdx piece = 0; + TextIdx hash = 0; + TextIdx lShift = 3; + TextIdx rShift = hashLen - lShift; + for (TextIdx i = 0; i < sliceLen - 1; ++i) { + TextIdx v = data[i]; + hash = (((hash << lShift) | (hash >> rShift)) & hashMask) ^ v; + } + TextIdx lShiftX = (lShift * (sliceLen - 1)) % hashLen; + TextIdx rShiftX = hashLen - lShiftX; + for (TextIdx i = 0; i < end; ++i) { + TextIdx v = data[i + sliceLen - 1]; + hash = (((hash << lShift) | (hash >> rShift)) & hashMask) ^ v; + + if (offsets[piece] == i) { + piece++; + } + TextIdx slot = hashHead[hash]; + while (slot != 0) { + Slot& item = map[slot]; + TextIdx start = item.offset; + bool miss = false; + for (TextIdx j = 0; j < sliceLen; ++j) { + if (data[i + j] != data[start + j]) { + miss = true; + break; + } + } + if (!miss) { + if (item.mark != piece) { + item.mark = piece; + item.presence++; + } + shortcut.push_back(slot); + break; + } + slot = item.next; + } + if (slot == 0) { + map.push_back({hashHead[hash], i, 1, piece}); + hashHead[hash] = hashSlot; + shortcut.push_back(hashSlot); + hashSlot++; + } + v = data[i]; + hash ^= ((v << lShiftX) | (v >> rShiftX)) & hashMask; + } + + /***************************************************************************** + * Build dictionary of specified size. + ****************************************************************************/ + SampleIdx a = 1; + TextIdx size = dryRun( + sliceLen, map.data(), shortcut.data(), end, end, a, ++piece); + /* Maximal output is smaller than target. */ + if (size <= targetSize) { + return createDictionary( + data, sliceLen, map.data(), shortcut.data(), end, end, a, ++piece); + } + + SampleIdx b = numSamples; + size = dryRun(sliceLen, map.data(), shortcut.data(), end, end, b, ++piece); + if (size == targetSize) { + return createDictionary( + data, sliceLen, map.data(), shortcut.data(), end, end, b, ++piece); + } + /* Run binary search. */ + if (size < targetSize) { + /* size(a) > targetSize > size(b) && a < m < b */ + while (a + 1 < b) { + SampleIdx m = static_cast<SampleIdx>((a + b) / 2); + size = dryRun( + sliceLen, map.data(), shortcut.data(), end, end, m, ++piece); + if (size < targetSize) { + b = m; + } else if (size > targetSize) { + a = m; + } else { + return createDictionary( + data, sliceLen, map.data(), shortcut.data(), end, end, b, ++piece); + } + } + } else { + a = b; + } + /* size(minPresence) > targetSize > size(minPresence + 1) */ + SampleIdx minPresence = a; + TextIdx c = 0; + TextIdx d = end; + /* size(a) < targetSize < size(b) && a < m < b */ + while (c + 1 < d) { + TextIdx m = (c + d) / 2; + size = dryRun( + sliceLen, map.data(), shortcut.data(), end, m, minPresence, ++piece); + if (size < targetSize) { + c = m; + } else if (size > targetSize) { + d = m; + } else { + return createDictionary(data, sliceLen, map.data(), shortcut.data(), end, + m, minPresence, ++piece); + } + } + + bool unrestricted = false; + if (minPresence <= 2 && !unrestricted) { + minPresence = 2; + c = end; + } + return createDictionary(data, sliceLen, map.data(), shortcut.data(), end, c, + minPresence, ++piece); +} diff --git a/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/sieve.h b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/sieve.h new file mode 100755 index 000000000..6c65dc8e0 --- /dev/null +++ b/roms/edk2/MdeModulePkg/Library/BrotliCustomDecompressLib/brotli/research/sieve.h @@ -0,0 +1,21 @@ +#ifndef BROTLI_RESEARCH_SIEVE_H_ +#define BROTLI_RESEARCH_SIEVE_H_ + +#include <cstddef> +#include <cstdint> +#include <string> +#include <vector> + +/** + * Generate a dictionary for given samples. + * + * @param dictionary_size_limit maximal dictionary size + * @param slice_len text slice size + * @param sample_sizes vector with sample sizes + * @param sample_data concatenated samples + * @return generated dictionary + */ +std::string sieve_generate(size_t dictionary_size_limit, size_t slice_len, + const std::vector<size_t>& sample_sizes, const uint8_t* sample_data); + +#endif // BROTLI_RESEARCH_SIEVE_H_ |
