asset compression
This commit is contained in:
parent
b9b9ecb84c
commit
30a283c1b3
57 changed files with 13237 additions and 8 deletions
136
implementations/Crafter.Asset-Compression.cpp
Normal file
136
implementations/Crafter.Asset-Compression.cpp
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
/*
|
||||
Crafter®.Asset
|
||||
Copyright (C) 2026 Catcrafts®
|
||||
catcrafts.net
|
||||
|
||||
This library is free software; you can redistribute it and/or
|
||||
modify it under the terms of the GNU Lesser General Public
|
||||
License version 3.0 as published by the Free Software Foundation;
|
||||
|
||||
This library is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||||
Lesser General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Lesser General Public
|
||||
License along with this library; if not, write to the Free Software
|
||||
Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
|
||||
*/
|
||||
|
||||
module;
|
||||
// Vendored GDeflate (Microsoft DirectStorage reference, Apache-2.0). Headers
|
||||
// pull libdeflate (MIT) via -Ilib/gdeflate/libdeflate. The C++ wrappers are
|
||||
// pulled inline because Crafter.Build's cFiles only handles .c TUs — folding
|
||||
// them into this module impl avoids adding a parallel cppFiles channel.
|
||||
#include "../lib/gdeflate/GDeflate.h"
|
||||
#include "../lib/gdeflate/GDeflateCompress.cpp"
|
||||
#include "../lib/gdeflate/GDeflateDecompress.cpp"
|
||||
|
||||
module Crafter.Asset;
|
||||
import std;
|
||||
|
||||
namespace Crafter::Compression {
|
||||
CompressedBlob CompressStreams(std::span<const std::span<const std::byte>> streams) {
|
||||
CompressedBlob blob;
|
||||
blob.regions.reserve(streams.size());
|
||||
|
||||
// First pass: compress each stream into its own buffer so we know the
|
||||
// exact final size. GDeflate::CompressBound is an upper bound; we'd
|
||||
// waste capacity if we appended the bound directly into a single
|
||||
// shared buffer.
|
||||
std::vector<std::vector<std::byte>> compressed;
|
||||
compressed.reserve(streams.size());
|
||||
|
||||
std::uint64_t totalSize = 0;
|
||||
for (const std::span<const std::byte>& stream : streams) {
|
||||
if (stream.empty()) {
|
||||
compressed.emplace_back();
|
||||
continue;
|
||||
}
|
||||
std::size_t boundSize = GDeflate::CompressBound(stream.size());
|
||||
std::vector<std::byte> out(boundSize);
|
||||
std::size_t actualSize = boundSize;
|
||||
bool ok = GDeflate::Compress(
|
||||
reinterpret_cast<std::uint8_t*>(out.data()),
|
||||
&actualSize,
|
||||
reinterpret_cast<const std::uint8_t*>(stream.data()),
|
||||
stream.size(),
|
||||
GDeflate::MaximumCompressionLevel,
|
||||
0);
|
||||
if (!ok) {
|
||||
throw std::runtime_error("GDeflate::Compress failed");
|
||||
}
|
||||
out.resize(actualSize);
|
||||
totalSize += actualSize;
|
||||
compressed.push_back(std::move(out));
|
||||
}
|
||||
|
||||
// Second pass: concatenate and build the region table.
|
||||
blob.bytes.reserve(totalSize);
|
||||
for (std::size_t i = 0; i < streams.size(); ++i) {
|
||||
RegionMeta r {
|
||||
.srcOffset = blob.bytes.size(),
|
||||
.compressedSize = compressed[i].size(),
|
||||
.decompressedSize = streams[i].size(),
|
||||
};
|
||||
blob.regions.push_back(r);
|
||||
blob.bytes.insert(blob.bytes.end(), compressed[i].begin(), compressed[i].end());
|
||||
}
|
||||
return blob;
|
||||
}
|
||||
|
||||
void WriteBlob(std::ostream& file, const CompressedBlob& blob) {
|
||||
std::uint32_t regionCount = static_cast<std::uint32_t>(blob.regions.size());
|
||||
file.write(reinterpret_cast<const char*>(®ionCount), sizeof(regionCount));
|
||||
if (regionCount > 0) {
|
||||
file.write(reinterpret_cast<const char*>(blob.regions.data()),
|
||||
regionCount * sizeof(RegionMeta));
|
||||
}
|
||||
std::uint64_t payloadSize = blob.bytes.size();
|
||||
file.write(reinterpret_cast<const char*>(&payloadSize), sizeof(payloadSize));
|
||||
if (payloadSize > 0) {
|
||||
file.write(reinterpret_cast<const char*>(blob.bytes.data()), payloadSize);
|
||||
}
|
||||
}
|
||||
|
||||
CompressedBlob ReadBlob(std::istream& file) {
|
||||
CompressedBlob blob;
|
||||
std::uint32_t regionCount = 0;
|
||||
file.read(reinterpret_cast<char*>(®ionCount), sizeof(regionCount));
|
||||
blob.regions.resize(regionCount);
|
||||
if (regionCount > 0) {
|
||||
file.read(reinterpret_cast<char*>(blob.regions.data()),
|
||||
regionCount * sizeof(RegionMeta));
|
||||
}
|
||||
std::uint64_t payloadSize = 0;
|
||||
file.read(reinterpret_cast<char*>(&payloadSize), sizeof(payloadSize));
|
||||
blob.bytes.resize(payloadSize);
|
||||
if (payloadSize > 0) {
|
||||
file.read(reinterpret_cast<char*>(blob.bytes.data()), payloadSize);
|
||||
}
|
||||
return blob;
|
||||
}
|
||||
|
||||
void DecompressCPU(const CompressedBlob& blob, std::span<const std::span<std::byte>> outputs) {
|
||||
if (outputs.size() != blob.regions.size()) {
|
||||
throw std::runtime_error("DecompressCPU: outputs.size() != regions.size()");
|
||||
}
|
||||
for (std::size_t i = 0; i < blob.regions.size(); ++i) {
|
||||
const RegionMeta& r = blob.regions[i];
|
||||
const std::span<std::byte>& out = outputs[i];
|
||||
if (out.size() != r.decompressedSize) {
|
||||
throw std::runtime_error("DecompressCPU: output size mismatch");
|
||||
}
|
||||
if (r.decompressedSize == 0) continue;
|
||||
bool ok = GDeflate::Decompress(
|
||||
reinterpret_cast<std::uint8_t*>(out.data()),
|
||||
r.decompressedSize,
|
||||
reinterpret_cast<const std::uint8_t*>(blob.bytes.data() + r.srcOffset),
|
||||
r.compressedSize,
|
||||
/*numWorkers=*/1);
|
||||
if (!ok) {
|
||||
throw std::runtime_error("GDeflate::Decompress failed");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -19,6 +19,35 @@ import std;
|
|||
using namespace Crafter;
|
||||
namespace fs = std::filesystem;
|
||||
|
||||
// CPU GDeflate roundtrip sanity test across the size boundaries from the
|
||||
// implementation plan. Returns 0 on pass, 1 on first byte-mismatch.
|
||||
static int RunCompressionRoundtrip() {
|
||||
const std::array<std::size_t, 5> sizes = { 1, 65535, 65536, 65537, 16 * 1024 * 1024 };
|
||||
std::mt19937_64 rng(0xC0FFEEu);
|
||||
for (std::size_t n : sizes) {
|
||||
std::vector<std::byte> input(n);
|
||||
for (std::size_t i = 0; i < n; ++i) {
|
||||
// Mix random bytes with a deterministic pattern so the codec is
|
||||
// exercised on both compressible and noisy regions.
|
||||
input[i] = static_cast<std::byte>((i * 0x9E3779B97F4A7C15ULL ^ rng()) & 0xFF);
|
||||
}
|
||||
std::array<std::span<const std::byte>, 1> streams = { std::span(input) };
|
||||
Compression::CompressedBlob blob = Compression::CompressStreams(streams);
|
||||
std::vector<std::byte> output(n);
|
||||
std::array<std::span<std::byte>, 1> outputs = { std::span(output) };
|
||||
Compression::DecompressCPU(blob, outputs);
|
||||
if (output != input) {
|
||||
std::cerr << "[FAIL] roundtrip size=" << n << "\n";
|
||||
return 1;
|
||||
}
|
||||
std::cout << "[ok] size=" << n
|
||||
<< " compressed=" << blob.bytes.size()
|
||||
<< " ratio=" << (double(blob.bytes.size()) / double(n)) << "\n";
|
||||
}
|
||||
std::cout << "All roundtrips passed.\n";
|
||||
return 0;
|
||||
}
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
// Parse arguments: crafter-asset <input_file> [output_file] [--format u8|f16]
|
||||
fs::path inputPath;
|
||||
|
|
@ -29,6 +58,9 @@ int main(int argc, char** argv) {
|
|||
std::vector<std::string> positional;
|
||||
for (int i = 1; i < argc; ++i) {
|
||||
std::string arg = argv[i];
|
||||
if (arg == "--test-compression") {
|
||||
return RunCompressionRoundtrip();
|
||||
}
|
||||
if (arg == "--format" || arg == "-f") {
|
||||
if (i + 1 >= argc) {
|
||||
std::cerr << "Error: --format requires a value (u8 or f16)\n";
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue