Overview
Comment: | OFLHAArchive: Skip compressed entries much faster |
---|---|
Downloads: | Tarball | ZIP archive | SQL archive |
Timelines: | family | ancestors | descendants | both | trunk |
Files: | files | file ages | folders |
SHA3-256: |
2e354552ad58025813cb8f204075c05f |
User & Date: | js on 2018-06-10 12:29:34 |
Other Links: | manifest | tags |
Context
2018-06-10
| ||
14:42 | ofzip: Show compression method for ZIP archives check-in: fd5dac720d user: js tags: trunk | |
12:29 | OFLHAArchive: Skip compressed entries much faster check-in: 2e354552ad user: js tags: trunk | |
2018-06-09
| ||
16:30 | OFObject: Change one #else to #elif check-in: 745d514023 user: js tags: trunk | |
Changes
Modified src/OFInflateStream.h from [3cdd10fe42] to [c2a72ef602].
︙ | ︙ | |||
77 78 79 80 81 82 83 | /*! * @brief Creates a new OFInflateStream with the specified underlying stream. * * @param stream The underlying stream to which compressed data is written or * from which compressed data is read * @return A new, autoreleased OFInflateStream */ | | | > | 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 | /*! * @brief Creates a new OFInflateStream with the specified underlying stream. * * @param stream The underlying stream to which compressed data is written or * from which compressed data is read * @return A new, autoreleased OFInflateStream */ + (instancetype)streamWithStream: (OF_KINDOF(OFStream *))stream; - (instancetype)init OF_UNAVAILABLE; /*! * @brief Initializes an already allocated OFInflateStream with the specified * underlying stream. * * @param stream The underlying stream to which compressed data is written or * from which compressed data is read * @return A initialized OFInflateStream */ - (instancetype)initWithStream: (OF_KINDOF(OFStream *))stream OF_DESIGNATED_INITIALIZER; @end OF_ASSUME_NONNULL_END |
Modified src/OFInflateStream.m from [84940553dd] to [673c1237d7].
︙ | ︙ | |||
168 169 170 171 172 173 174 | for (uint16_t i = 0; i <= 31; i++) lengths[i] = 5; fixedDistTree = of_huffman_tree_construct(lengths, 32); } | | | | 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 | for (uint16_t i = 0; i <= 31; i++) lengths[i] = 5; fixedDistTree = of_huffman_tree_construct(lengths, 32); } + (instancetype)streamWithStream: (OF_KINDOF(OFStream *))stream { return [[[self alloc] initWithStream: stream] autorelease]; } - (instancetype)init { OF_INVALID_INIT_METHOD } - (instancetype)initWithStream: (OF_KINDOF(OFStream *))stream { self = [super init]; @try { _stream = [stream retain]; /* 0-7 address the bit, 8 means fetch next byte */ |
︙ | ︙ |
Modified src/OFLHAArchive.m from [a7a157c7cf] to [4ab39c71a2].
︙ | ︙ | |||
40 41 42 43 44 45 46 | #import "OFTruncatedDataException.h" #define LHSTREAM_BUFFER_SIZE 4096 @interface OFLHAArchive_LHStream: OFStream { @public | | > | > > | | | | 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 | #import "OFTruncatedDataException.h" #define LHSTREAM_BUFFER_SIZE 4096 @interface OFLHAArchive_LHStream: OFStream { @public OF_KINDOF(OFStream *) _stream; uint8_t _distanceBits, _dictionaryBits; unsigned char _buffer[LHSTREAM_BUFFER_SIZE]; uint32_t _bytesConsumed; uint16_t _bufferIndex, _bufferLength; uint8_t _byte; uint8_t _bitIndex, _savedBitsLength; uint16_t _savedBits; unsigned char *_slidingWindow; uint32_t _slidingWindowIndex, _slidingWindowMask; int _state; uint16_t _symbolsLeft; struct of_huffman_tree *_codeLenTree, *_litLenTree, *_distTree; struct of_huffman_tree *_treeIter; uint16_t _codesCount, _codesReceived; bool _currentIsExtendedLength, _skip; uint8_t *_codesLengths; uint16_t _length; uint32_t _distance; } - (instancetype)of_initWithStream: (OF_KINDOF(OFStream *))stream distanceBits: (uint8_t)distanceBits dictionaryBits: (uint8_t)dictionaryBits; @end @interface OFLHAArchive_FileReadStream: OFStream <OFReadyForReadingObserving> { OF_KINDOF(OFStream *) _stream; OF_KINDOF(OFStream *) _decompressedStream; OFLHAArchiveEntry *_entry; uint32_t _toRead, _bytesConsumed; uint16_t _CRC16; bool _atEndOfStream; } - (instancetype)of_initWithStream: (OF_KINDOF(OFStream *))stream entry: (OFLHAArchiveEntry *)entry; - (void)of_skip; @end enum state { STATE_BLOCK_HEADER, STATE_CODE_LEN_CODES_COUNT, |
︙ | ︙ | |||
111 112 113 114 115 116 117 118 119 120 121 122 123 124 | if (stream->_bufferIndex < stream->_bufferLength) stream->_byte = stream->_buffer[stream->_bufferIndex++]; else { size_t length = [stream->_stream readIntoBuffer: stream->_buffer length: LHSTREAM_BUFFER_SIZE]; if OF_UNLIKELY (length < 1) { stream->_savedBits = ret; stream->_savedBitsLength = i; return false; } | > > | 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 | if (stream->_bufferIndex < stream->_bufferLength) stream->_byte = stream->_buffer[stream->_bufferIndex++]; else { size_t length = [stream->_stream readIntoBuffer: stream->_buffer length: LHSTREAM_BUFFER_SIZE]; stream->_bytesConsumed += (uint32_t)length; if OF_UNLIKELY (length < 1) { stream->_savedBits = ret; stream->_savedBitsLength = i; return false; } |
︙ | ︙ | |||
279 280 281 282 283 284 285 | [_stream release]; _stream = nil; } @end @implementation OFLHAArchive_LHStream | | | 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 | [_stream release]; _stream = nil; } @end @implementation OFLHAArchive_LHStream - (instancetype)of_initWithStream: (OF_KINDOF(OFStream *))stream distanceBits: (uint8_t)distanceBits dictionaryBits: (uint8_t)dictionaryBits { self = [super init]; @try { _stream = [stream retain]; |
︙ | ︙ | |||
598 599 600 601 602 603 604 605 606 607 608 609 610 611 | * We must also unread the buffer, in case this was the * last block and something else follows, e.g. another * LHA header. */ [_stream unreadFromBuffer: _buffer + _bufferIndex length: _bufferLength - _bufferIndex]; _bufferIndex = _bufferLength = 0; return bytesWritten; } if OF_UNLIKELY (length == 0) return bytesWritten; | > | 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 | * We must also unread the buffer, in case this was the * last block and something else follows, e.g. another * LHA header. */ [_stream unreadFromBuffer: _buffer + _bufferIndex length: _bufferLength - _bufferIndex]; _bytesConsumed -= _bufferLength - _bufferIndex; _bufferIndex = _bufferLength = 0; return bytesWritten; } if OF_UNLIKELY (length == 0) return bytesWritten; |
︙ | ︙ | |||
685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 | { if (_stream == nil) @throw [OFNotOpenException exceptionWithObject: self]; return ([_stream isAtEndOfStream] && _bufferLength - _bufferIndex == 0 && _state == STATE_BLOCK_HEADER); } - (void)close { /* Give back our buffer to the stream, in case it's shared */ [_stream unreadFromBuffer: _buffer + _bufferIndex length: _bufferLength - _bufferIndex]; _bufferIndex = _bufferLength = 0; [_stream release]; _stream = nil; [super close]; } @end @implementation OFLHAArchive_FileReadStream | > > > > > > > > > > > | | > > > > | | | | | < < | < < < < > > > > > > > > > > > > | | | | > > > | > > > > | > | > > > > | | > > > > > > > > > | > > | | | > > > > > > > > | > > | > > > > | > < < | < < < < | < < < < < < < < < | < < | < < < < < | 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 | { if (_stream == nil) @throw [OFNotOpenException exceptionWithObject: self]; return ([_stream isAtEndOfStream] && _bufferLength - _bufferIndex == 0 && _state == STATE_BLOCK_HEADER); } - (int)fileDescriptorForReading { return [_stream fileDescriptorForReading]; } - (bool)hasDataInReadBuffer { return ([super hasDataInReadBuffer] || [_stream hasDataInReadBuffer]); } - (void)close { /* Give back our buffer to the stream, in case it's shared */ [_stream unreadFromBuffer: _buffer + _bufferIndex length: _bufferLength - _bufferIndex]; _bytesConsumed -= _bufferLength - _bufferIndex; _bufferIndex = _bufferLength = 0; [_stream release]; _stream = nil; [super close]; } @end @implementation OFLHAArchive_FileReadStream - (instancetype)of_initWithStream: (OF_KINDOF(OFStream *))stream entry: (OFLHAArchiveEntry *)entry { self = [super init]; @try { OFString *method; _stream = [stream retain]; method = [entry method]; if ([method isEqual: @"-lh4-"] || [method isEqual: @"-lh5-"]) _decompressedStream = [[OFLHAArchive_LHStream alloc] of_initWithStream: stream distanceBits: 4 dictionaryBits: 14]; else if ([method isEqual: @"-lh6-"]) _decompressedStream = [[OFLHAArchive_LHStream alloc] of_initWithStream: stream distanceBits: 5 dictionaryBits: 16]; else if ([method isEqual: @"-lh7-"]) _decompressedStream = [[OFLHAArchive_LHStream alloc] of_initWithStream: stream distanceBits: 5 dictionaryBits: 17]; else _decompressedStream = [stream retain]; _entry = [entry copy]; _toRead = [entry uncompressedSize]; } @catch (id e) { [self release]; @throw e; } return self; } - (void)dealloc { [self close]; [_stream release]; [_decompressedStream release]; [_entry release]; [super dealloc]; } - (bool)lowlevelIsAtEndOfStream { if (_stream == nil) @throw [OFNotOpenException exceptionWithObject: self]; return _atEndOfStream; } - (size_t)lowlevelReadIntoBuffer: (void *)buffer length: (size_t)length { size_t ret; if (_stream == nil) @throw [OFNotOpenException exceptionWithObject: self]; if (_atEndOfStream) return 0; if (length > _toRead) length = _toRead; ret = [_decompressedStream readIntoBuffer: buffer length: length]; _toRead -= ret; _CRC16 = of_crc16(_CRC16, buffer, ret); if (_toRead == 0) { _atEndOfStream = true; if (_CRC16 != [_entry CRC16]) @throw [OFChecksumFailedException exception]; } return ret; } - (bool)hasDataInReadBuffer { return ([super hasDataInReadBuffer] || [_decompressedStream hasDataInReadBuffer]); } - (int)fileDescriptorForReading { return [_decompressedStream fileDescriptorForReading]; } - (void)of_skip { OF_KINDOF(OFStream *) stream; uint32_t toRead; if (_stream == nil || _toRead == 0) return; stream = _stream; toRead = _toRead; /* * Get the number of consumed bytes and directly read from the * compressed stream, to make skipping much faster. */ if ([_decompressedStream isKindOfClass: [OFLHAArchive_LHStream class]]) { OFLHAArchive_LHStream *LHStream = _decompressedStream; [LHStream close]; toRead = [_entry compressedSize] - LHStream->_bytesConsumed; stream = _stream; } if ([stream isKindOfClass: [OFSeekableStream class]] && (sizeof(of_offset_t) > 4 || toRead < INT32_MAX)) [stream seekToOffset: (of_offset_t)toRead whence: SEEK_CUR]; else { while (toRead > 0) { char buffer[512]; size_t min = toRead; if (min > 512) min = 512; toRead -= [stream readIntoBuffer: buffer length: min]; } } _toRead = 0; } - (void)close { [self of_skip]; [_stream release]; _stream = nil; [_decompressedStream release]; _decompressedStream = nil; [super close]; } @end |
Modified src/OFZIPArchive.m from [56c946d958] to [f05c35edb7].
︙ | ︙ | |||
39 40 41 42 43 44 45 46 47 48 49 50 51 52 | #import "OFInvalidArgumentException.h" #import "OFInvalidFormatException.h" #import "OFNotImplementedException.h" #import "OFNotOpenException.h" #import "OFOpenItemFailedException.h" #import "OFOutOfRangeException.h" #import "OFSeekFailedException.h" #import "OFUnsupportedVersionException.h" /* * FIXME: Current limitations: * - Split archives are not supported. * - Encrypted files cannot be read. */ | > | 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 | #import "OFInvalidArgumentException.h" #import "OFInvalidFormatException.h" #import "OFNotImplementedException.h" #import "OFNotOpenException.h" #import "OFOpenItemFailedException.h" #import "OFOutOfRangeException.h" #import "OFSeekFailedException.h" #import "OFTruncatedDataException.h" #import "OFUnsupportedVersionException.h" /* * FIXME: Current limitations: * - Split archives are not supported. * - Encrypted files cannot be read. */ |
︙ | ︙ | |||
71 72 73 74 75 76 77 | - (instancetype)initWithStream: (OFStream *)stream; - (bool)matchesEntry: (OFZIPArchiveEntry *)entry; @end @interface OFZIPArchive_FileReadStream: OFStream { | > | | 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 | - (instancetype)initWithStream: (OFStream *)stream; - (bool)matchesEntry: (OFZIPArchiveEntry *)entry; @end @interface OFZIPArchive_FileReadStream: OFStream { OF_KINDOF(OFStream *) _stream; OF_KINDOF(OFStream *) _decompressedStream; OFZIPArchiveEntry *_entry; uint64_t _toRead; uint32_t _CRC32; bool _atEndOfStream; } - (instancetype)of_initWithStream: (OFStream *)stream |
︙ | ︙ | |||
774 775 776 777 778 779 780 781 782 783 784 785 786 787 | [_entry release]; [super dealloc]; } - (bool)lowlevelIsAtEndOfStream { return _atEndOfStream; } - (size_t)lowlevelReadIntoBuffer: (void *)buffer length: (size_t)length { size_t ret; | > > > | 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 | [_entry release]; [super dealloc]; } - (bool)lowlevelIsAtEndOfStream { if (_stream == nil) @throw [OFNotOpenException exceptionWithObject: self]; return _atEndOfStream; } - (size_t)lowlevelReadIntoBuffer: (void *)buffer length: (size_t)length { size_t ret; |
︙ | ︙ | |||
799 800 801 802 803 804 805 | if ((uint64_t)length > _toRead) length = (size_t)_toRead; ret = [_decompressedStream readIntoBuffer: buffer length: length]; | < < < | > > > > > > > > > > > > > > > > > | 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 | if ((uint64_t)length > _toRead) length = (size_t)_toRead; ret = [_decompressedStream readIntoBuffer: buffer length: length]; _toRead -= ret; _CRC32 = of_crc32(_CRC32, buffer, ret); if (_toRead == 0) { _atEndOfStream = true; if (~_CRC32 != [_entry CRC32]) @throw [OFChecksumFailedException exception]; } return ret; } - (bool)hasDataInReadBuffer { return ([super hasDataInReadBuffer] || [_decompressedStream hasDataInReadBuffer]); } - (int)fileDescriptorForReading { return [_decompressedStream fileDescriptorForReading]; } - (void)close { [_stream release]; _stream = nil; [_decompressedStream release]; _decompressedStream = nil; [super close]; } @end @implementation OFZIPArchive_FileWriteStream - (instancetype)initWithStream: (OFStream *)stream |
︙ | ︙ |