/* Copyright (C) Teemu Suutari */ #include #include #include "DMSDecompressor.hpp" #include "HuffmanDecoder.hpp" #include "DynamicHuffmanDecoder.hpp" #include "InputStream.hpp" #include "OutputStream.hpp" #include "common/MemoryBuffer.hpp" #include "common/CRC16.hpp" #include "common/OverflowCheck.hpp" #include "common/Common.hpp" namespace ancient::internal { bool DMSDecompressor::detectHeader(uint32_t hdr) noexcept { return hdr==FourCC("DMS!"); } std::shared_ptr DMSDecompressor::create(const Buffer &packedData,bool exactSizeKnown,bool verify) { return std::make_shared(packedData,verify); } DMSDecompressor::DMSDecompressor(const Buffer &packedData,bool verify) : _packedData(packedData) { uint32_t hdr=packedData.readBE32(0); if (!detectHeader(hdr) || packedData.size()<56) throw InvalidFormatError(); if (verify && CRC16(packedData,4,50,0)!=packedData.readBE16(54)) throw VerificationError(); uint16_t info=packedData.readBE16(10); _isObsfuscated=info&2; // using 16 bit key is not encryption, it is obsfuscation _isHD=info&16; if (info&32) throw InvalidFormatError(); // MS-DOS disk // packed data in header is useless, as is rawsize and track numbers // they are not always filled if (packedData.readBE16(50)>6) throw InvalidFormatError(); // either FMS or unknown // now calculate the real packed size, including headers uint32_t offset=56; uint32_t accountedSize=0; uint32_t lastTrackSize=0; uint32_t numTracks=0; uint32_t minTrack=80; uint32_t prevTrack=0; while (offset+206) throw InvalidFormatError(); static const uint32_t contextSizes[7]={0,0,256,16384,16384,4096,8192}; _contextBufferSize=std::max(_contextBufferSize,contextSizes[mode]); uint8_t flags=_packedData.read8(offset+12); if ((mode>=2 && mode<=4) || (mode>=5 && (flags&4))) { _tmpBufferSize=std::max(_tmpBufferSize,uint32_t(_packedData.readBE16(offset+8))); } uint32_t packedChunkLength=packedData.readBE16(offset+6); if (OverflowCheck::sum(offset,20U,packedChunkLength)>packedData.size()) throw InvalidFormatError(); if (verify && CRC16(packedData,offset+20,packedChunkLength,0)!=packedData.readBE16(offset+16)) throw VerificationError(); if (trackNo<80) { if (trackNo>=numTracks) lastTrackSize=_packedData.readBE16(offset+10); minTrack=std::min(minTrack,trackNo); numTracks=std::max(numTracks,trackNo); prevTrack=trackNo; } offset+=packedChunkLength+20; accountedSize+=packedChunkLength; // this is the real exit critea, unfortunately if (trackNo>=79 && trackNo<0x8000U) break; } uint32_t trackSize=(_isHD)?22528:11264; _rawOffset=minTrack*trackSize; if (minTrack>=numTracks) throw InvalidFormatError(); _minTrack=minTrack; _rawSize=(numTracks-minTrack)*trackSize+lastTrackSize; _imageSize=trackSize*80; _packedSize=offset; if (_packedSize>getMaxPackedSize()) throw InvalidFormatError(); } DMSDecompressor::~DMSDecompressor() { // nothing needed } const std::string &DMSDecompressor::getName() const noexcept { static std::string name="DMS: Disk Masher System"; return name; } size_t DMSDecompressor::getPackedSize() const noexcept { return _packedSize; } size_t DMSDecompressor::getRawSize() const noexcept { return _rawSize; } size_t DMSDecompressor::getImageSize() const noexcept { return _imageSize; } size_t DMSDecompressor::getImageOffset() const noexcept { return _rawOffset; } void DMSDecompressor::decompressImpl(Buffer &rawData,bool verify) { uint32_t restartPosition=0; if (!_isObsfuscated) { decompressImpl(rawData,verify,restartPosition); } else { while (restartPosition<0x20000U) { // more than single run here is really rare. It means that first track CRC succeeds // but later something else fails try { decompressImpl(rawData,verify,restartPosition); return; } catch (const Buffer::Error &) { // just continue } catch (const Decompressor::Error &) { // just continue } restartPosition++; } throw DecompressionError(); } } // TODO: Too much state for a single method. too convoluted // needs to be split void DMSDecompressor::decompressImpl(Buffer &rawData,bool verify,uint32_t &restartPosition) { if (rawData.size()<_rawSize) throw DecompressionError(); MemoryBuffer contextBuffer(_contextBufferSize); MemoryBuffer tmpBuffer(_tmpBufferSize); uint32_t limitedDecompress=~0U; class UnObsfuscator { public: UnObsfuscator(ForwardInputStream &inputStream) : _inputStream(inputStream) { // nothing needed } uint8_t readByte() { if (_inputStream.eof()) throw ShortInputError(); uint8_t ch=_inputStream.readByte(); if (!_obsfuscate) { return ch; } else { uint8_t ret=ch^_passAccumulator; _passAccumulator=(_passAccumulator>>1)+uint16_t(ch); return ret; } } void setCode(uint16_t passAccumulator) { _passAccumulator=passAccumulator; } void setObsfuscate(bool obsfuscate) { _obsfuscate=obsfuscate; } bool eof() const { return _inputStream.getOffset()==_inputStream.getEndOffset(); } // not cool, but works (does not need wraparound check) bool eofMinus1() const { return _inputStream.getOffset()+1==_inputStream.getEndOffset(); } bool eofMinus1Plus() const { return _inputStream.getOffset()+1>=_inputStream.getEndOffset(); } bool eofMinus2Plus() const { return _inputStream.getOffset()+2>=_inputStream.getEndOffset(); } private: ForwardInputStream &_inputStream; bool _obsfuscate=false; uint16_t _passAccumulator=0; }; ForwardInputStream inputStream(_packedData,0,0); UnObsfuscator inputUnObsfuscator(inputStream); MSBBitReader bitReader(inputUnObsfuscator); auto initInputStream=[&](const Buffer &buffer,uint32_t start,uint32_t length,bool obsfuscate) { inputStream=ForwardInputStream(buffer,start,OverflowCheck::sum(start,length)); inputUnObsfuscator.setObsfuscate(obsfuscate); bitReader.reset(); }; auto finishStream=[&]() { if (_isObsfuscated && limitedDecompress==~0U) while (!inputUnObsfuscator.eof()) inputUnObsfuscator.readByte(); }; auto readBits=[&](uint32_t count)->uint32_t { return bitReader.readBits8(count); }; auto readBit=[&]()->uint32_t { return bitReader.readBits8(1); }; ForwardOutputStream outputStream(rawData,0,0); auto initOutputStream=[&](Buffer &buffer,uint32_t start,uint32_t length) { outputStream=ForwardOutputStream(buffer,start,OverflowCheck::sum(start,length)); }; // fill unused tracks with zeros std::memset(rawData.data(),0,_rawSize); auto checksum=[](const uint8_t *src,uint32_t srcLength)->uint16_t { uint16_t ret=0; for (uint32_t i=0;iuint32_t { // hacky, hacky, hacky while (!inputUnObsfuscator.eof()) { if (outputStream.getOffset()>=limitedDecompress) return 0; if (lastCharMissing && inputUnObsfuscator.eofMinus1()) { if (outputStream.getOffset()+1!=outputStream.getEndOffset()) throw DecompressionError(); return 1; } uint8_t ch=inputUnObsfuscator.readByte(); uint32_t count=1; if (ch==0x90U) { if (inputUnObsfuscator.eof()) throw DecompressionError(); if (lastCharMissing && inputUnObsfuscator.eofMinus1()) { // only possible way this can work if (outputStream.getOffset()+1!=outputStream.getEndOffset()) throw DecompressionError(); count=0; } else count=inputUnObsfuscator.readByte(); if (!count) { count=1; } else { if (inputUnObsfuscator.eof()) throw DecompressionError(); if (lastCharMissing && inputUnObsfuscator.eofMinus1()) { if (count==0xffU || outputStream.getOffset()+count!=outputStream.getEndOffset()) throw DecompressionError(); return count; } ch=inputUnObsfuscator.readByte(); } if (count==0xffU) { if (inputUnObsfuscator.eofMinus1Plus()) throw DecompressionError(); if (lastCharMissing && inputUnObsfuscator.eofMinus2Plus()) { count=uint32_t(outputStream.getEndOffset()-outputStream.getOffset()); } else { count=uint32_t(inputUnObsfuscator.readByte())<<8; count|=inputUnObsfuscator.readByte(); } } } for (uint32_t i=0;i> deepDecoder; auto initContext=[&]() { if (doInitContext) { if (_contextBufferSize) std::memset(contextBuffer.data(),0,_contextBufferSize); quickContextLocation=251; mediumContextLocation=16318; deepContextLocation=16324; deepDecoder.reset(); heavyContextLocation=0; doInitContext=false; } }; auto unpackQuick=[&]() { initContext(); while (!outputStream.eof()) { if (outputStream.getOffset()>=limitedDecompress) return; if (readBits(1)) { outputStream.writeByte(contextBufferPtr[quickContextLocation++]=readBits(8)); } else { uint32_t count=readBits(2)+2; uint8_t offset=quickContextLocation-readBits(8)-1; for (uint32_t i=0;iuint32_t { return (((code<uint32_t { return (uint32_t(lengthTable[code])<<8)| uint32_t(((code<=limitedDecompress) return; if (readBits(1)) { outputStream.writeByte(contextBufferPtr[mediumContextLocation++]=readBits(8)); mediumContextLocation&=0x3fffU; } else { uint32_t code=readBits(8); uint32_t count=lengthTable[code]+3; uint32_t tmp=decodeLengthValueFull(decodeLengthValueHalf(code)); uint32_t offset=mediumContextLocation-tmp-1; for (uint32_t i=0;i>(); while (!outputStream.eof()) { if (outputStream.getOffset()>=limitedDecompress) return; uint32_t symbol=deepDecoder->decode(readBit); if (deepDecoder->getMaxFrequency()==0x8000U) deepDecoder->halve(); deepDecoder->update(symbol); if (symbol<256) { outputStream.writeByte(contextBufferPtr[deepContextLocation++]=symbol); deepContextLocation&=0x3fffU; } else { uint32_t count=symbol-253; // minimum repeat is 3 uint32_t offset=deepContextLocation-decodeLengthValueFull(readBits(8))-1; for (uint32_t i=0;i> symbolDecoder,offsetDecoder; bool heavyLastInitialized=false; // this is part of initContext on some implementations. screwy!!! uint32_t heavyLastOffset; auto unpackHeavy=[&](bool initTables,bool use8kDict) { initContext(); // well, this works. Why this works? dunno if (!heavyLastInitialized) { heavyLastOffset=use8kDict?0U:~0U; heavyLastInitialized=true; } auto readTable=[&](std::unique_ptr> &decoder,uint32_t countBits,uint32_t valueBits) { decoder=std::make_unique>(); uint32_t count=readBits(countBits); if (count) { uint8_t lengthBuffer[512]; // in order to speed up the deObsfuscation, do not send the hopeless // data into slow CreateOrderlyHuffmanTable uint64_t sum=0; for (uint32_t i=0;i(uint64_t(1U)<<32)) throw DecompressionError(); } lengthBuffer[i]=bits; } decoder->createOrderlyHuffmanTable(lengthBuffer,count); } else { uint32_t index=readBits(countBits); decoder->setEmpty(index); } }; if (initTables) { readTable(symbolDecoder,9,5); readTable(offsetDecoder,5,4); } uint32_t mask=use8kDict?0x1fffU:0xfffU; uint32_t bitLength=use8kDict?14:13; while (!outputStream.eof()) { if (outputStream.getOffset()>=limitedDecompress) return; uint32_t symbol=symbolDecoder->decode(readBit); if (symbol<256) { outputStream.writeByte(contextBufferPtr[heavyContextLocation++]=symbol); heavyContextLocation&=mask; } else { uint32_t count=symbol-253; // minimum repeat is 3 uint32_t offsetLength=offsetDecoder->decode(readBit); uint32_t rawOffset=heavyLastOffset; if (offsetLength!=bitLength) { if (offsetLength) rawOffset=(1<<(offsetLength-1))|readBits(offsetLength-1); else rawOffset=0; heavyLastOffset=rawOffset; } uint32_t offset=heavyContextLocation-rawOffset-1; for (uint32_t i=0;i=0x8000U) { initInputStream(_packedData,packedOffset+20,packedChunkLength,_isObsfuscated); finishStream(); continue; } if (rawChunkLength>trackLength) throw DecompressionError(); if (trackNo>80) throw DecompressionError(); // should not happen, already excluded uint32_t dataOffset=trackNo*trackLength; if (_rawOffset>dataOffset) throw DecompressionError(); // this is screwy, but it is, what it is auto processBlock=[&](bool doRLE,auto func,auto&&... params) { initInputStream(_packedData,packedOffset+20,packedChunkLength,_isObsfuscated); if (doRLE) { try { initOutputStream(tmpBuffer,0,tmpChunkLength); func(params...); finishStream(); initInputStream(tmpBuffer,0,tmpChunkLength,false); initOutputStream(rawData,dataOffset-_rawOffset,rawChunkLength); unRLE(false); } catch (const ShortInputError &) { // if this error happens on repeat/offset instead of char, though luck :( // missing last char on src we can fix :) initInputStream(tmpBuffer,0,tmpChunkLength,false); initOutputStream(rawData,dataOffset-_rawOffset,rawChunkLength); uint32_t missingNo=unRLE(true); if (missingNo) { uint32_t protoSum=checksum(&rawData[dataOffset-_rawOffset],rawChunkLength-missingNo); uint32_t fileSum=_packedData.readBE16(packedOffset+14); uint8_t ch=((fileSum>=protoSum)?fileSum-protoSum:(0x10000U+fileSum-protoSum))/missingNo; for (uint32_t i=0;i