From d1a97fec8d8976680f77506b2b82ed08e99763cc Mon Sep 17 00:00:00 2001 From: ponchio Date: Sun, 10 Oct 2004 17:19:42 +0000 Subject: [PATCH] Added compression and debugged. --- apps/nexus/nexus.cpp | 10 +-- apps/nexus/nexus.h | 4 +- apps/nexus/nexusmt.cpp | 4 +- apps/nexus/nexusmt.h | 2 +- apps/nexus/nxsedit.cpp | 38 +++++++++-- apps/nexus/patch.cpp | 71 +++++++++++++------- apps/nexus/patch.h | 8 ++- apps/nexus/patchserver.cpp | 128 ++++++++++++++++++++++++++---------- apps/nexus/patchserver.h | 7 +- apps/nexus/voronoichain.cpp | 9 ++- apps/nexus/voronoinxs.cpp | 24 +++++-- 11 files changed, 222 insertions(+), 83 deletions(-) diff --git a/apps/nexus/nexus.cpp b/apps/nexus/nexus.cpp index f165fc11..bf4c5418 100644 --- a/apps/nexus/nexus.cpp +++ b/apps/nexus/nexus.cpp @@ -11,7 +11,7 @@ Nexus::~Nexus() { Close(); } -bool Nexus::Create(const string &file, Signature sig) { +bool Nexus::Create(const string &file, Signature sig, unsigned int c_size) { index_file = fopen((file + ".nxs").c_str(), "wb+"); if(!index_file) { cerr << "Could not create file: " << file << ".nxs\n"; @@ -25,7 +25,7 @@ bool Nexus::Create(const string &file, Signature sig) { index.clear(); - chunk_size = 1024; + chunk_size = c_size; if(!patches.Create(file + ".nxp", signature, chunk_size)) { cerr << "Could not create file: " << file << ".nxp" << endl; @@ -96,6 +96,9 @@ bool Nexus::Load(const string &file, bool readonly) { } void Nexus::Close() { + patches.Close(); + borders.Close(); + if(!index_file) return; rewind(index_file); @@ -131,9 +134,6 @@ void Nexus::Close() { fclose(index_file); index_file = NULL; - - patches.Close(); - borders.Close(); } Patch &Nexus::GetPatch(unsigned int patch, bool flush) { diff --git a/apps/nexus/nexus.h b/apps/nexus/nexus.h index a4d353c5..24aee63c 100644 --- a/apps/nexus/nexus.h +++ b/apps/nexus/nexus.h @@ -50,7 +50,8 @@ class Nexus { Nexus(); virtual ~Nexus(); - bool Create(const std::string &filename, Signature signature); + bool Create(const std::string &filename, Signature signature, + unsigned int chunk_size = 1024); virtual bool Load(const std::string &filename, bool readonly = false); virtual void Close(); @@ -77,6 +78,7 @@ class Nexus { /* Nexus data */ //BE CAREFUL: this 2 members get replicated into patchserver + //TODO fix this nasty thing it is dangerous as it is. Signature signature; unsigned int chunk_size; diff --git a/apps/nexus/nexusmt.cpp b/apps/nexus/nexusmt.cpp index a6cce785..507b55c5 100644 --- a/apps/nexus/nexusmt.cpp +++ b/apps/nexus/nexusmt.cpp @@ -41,8 +41,8 @@ NexusMt::NexusMt(): vbo(VBO_AUTO), vbo_size(0), NexusMt::~NexusMt() {} -bool NexusMt::Load(const string &filename) { - if(!Nexus::Load(filename)) return false; +bool NexusMt::Load(const string &filename, bool readonly) { + if(!Nexus::Load(filename, readonly)) return false; LoadHistory(); use_colors = false; diff --git a/apps/nexus/nexusmt.h b/apps/nexus/nexusmt.h index b5f1f35d..9775067b 100644 --- a/apps/nexus/nexusmt.h +++ b/apps/nexus/nexusmt.h @@ -83,7 +83,7 @@ class NexusMt: public Nexus { NexusMt(); ~NexusMt(); - bool Load(const std::string &filename); + bool Load(const std::string &filename, bool readonly = true); bool InitGL(); void Render(); diff --git a/apps/nexus/nxsedit.cpp b/apps/nexus/nxsedit.cpp index 3ee21e53..1338e450 100644 --- a/apps/nexus/nxsedit.cpp +++ b/apps/nexus/nxsedit.cpp @@ -30,6 +30,8 @@ int main(int argc, char *argv[]) { string output; string plysource; bool info = false; + unsigned int ram_size = 128000000; + unsigned int chunk_size = 0; unsigned int add = 0; bool add_strip = false; @@ -53,7 +55,7 @@ int main(int argc, char *argv[]) { float qtexture = 0; int option; - while((option = getopt(argc, argv, "io:a:r:zxv:n:c:t:")) != EOF) { + while((option = getopt(argc, argv, "io:a:r:zxv:n:k:t:b:c:")) != EOF) { switch(option) { case 'i': info = true; break; case 'o': output = optarg; break; @@ -135,7 +137,7 @@ int main(int argc, char *argv[]) { return -1; } break; - case 'c': qcolor = atof(optarg); + case 'k': qcolor = atof(optarg); if(qcolor == 0) { cerr << "Invalid value for quantization: " << optarg << endl; return -1; @@ -147,7 +149,18 @@ int main(int argc, char *argv[]) { return -1; } break; - + case 'b': ram_size = atoi(optarg); + if(ram_size == 0) { + cerr << "Invalid ram_size: " << optarg << endl; + return -1; + } + break; + case 'c': chunk_size = atoi(optarg); + if(chunk_size == 0) { + cerr << "Invalid chunk_size: " << optarg << endl; + return -1; + } + break; default: cerr << "Unknown option: " << (char)option << endl; return -1; } @@ -177,11 +190,13 @@ int main(int argc, char *argv[]) { Nexus nexus; - if(!nexus.Load(input)) { + nexus.patches.SetRamBufferSize(ram_size); + if(!nexus.Load(input, true)) { cerr << "Could not open nexus file: " << input << ".mt\n"; return -1; } + //Sanity tests if(remove_strip && !(nexus.signature & NXS_STRIP)) { cerr << "Nexus file does not have strips\n"; @@ -226,7 +241,9 @@ int main(int argc, char *argv[]) { << "\n\tData : " << (int)((nexus.signature&NXS_DATA32) !=0) << "\n\n\tVertices: " << nexus.totvert << "\tFaces: " << nexus.totface - << "\tPatches: " << nexus.index.size() << "\n\n"; + << "\tPatches: " << nexus.index.size() + << "\nChunk size " << nexus.chunk_size << endl; + } //determine if we must proceed: @@ -239,15 +256,24 @@ int main(int argc, char *argv[]) { unsigned int signature = nexus.signature; signature |= add; signature &= ~remove; + if(compress) signature |= NXS_COMPRESSED; + if(uncompress) signature &= ~NXS_COMPRESSED; if(!output.size()) output = input + getSuffix(signature); + cerr << "Writing to nexus: " << output << endl; + Nexus out; - if(!out.Create(output, (Signature)signature)) { + out.patches.SetRamBufferSize(ram_size); + if(!chunk_size) + chunk_size = nexus.patches.chunk_size; + + if(!out.Create(output, (Signature)signature, chunk_size)) { cerr << "Could not open output: " << output << endl; return -1; } + for(unsigned int patch = 0; patch < nexus.index.size(); patch++) { Nexus::PatchInfo &src_entry = nexus.index[patch]; Patch src_patch = nexus.GetPatch(patch); diff --git a/apps/nexus/patch.cpp b/apps/nexus/patch.cpp index 968ab12b..5a33df40 100644 --- a/apps/nexus/patch.cpp +++ b/apps/nexus/patch.cpp @@ -1,7 +1,11 @@ #include "patch.h" - +#include +#include +using namespace std; using namespace nxs; +static double wrkmem[LZO1X_999_MEM_COMPRESS/sizeof(double) +1]; + void pad(unsigned int &size) { while(size&0x3) size++; } @@ -46,27 +50,6 @@ void Patch::Init(Signature signature, dstart = tstart + nv; else dstart = tstart; - - /* cstart = nv * sizeof(float) * 3; - if(signature & NXS_COLORS) - nstart = cstart + nv * sizeof(unsigned int); - else - nstart = cstart; - - if(signature & NXS_NORMALS_SHORT) - tstart = nstart + nv * sizeof(short) * 4; - else if(signature & NXS_NORMALS_FLOAT) - tstart = nstart + nv * sizeof(float) * 3; - else - tstart = nstart; - - if(signature & NXS_TEXTURES_SHORT) - dstart = tstart + nv * sizeof(short) * 2; - else if(signature & NXS_TEXTURES_FLOAT) - dstart = tstart + nv * sizeof(float) * 2; - else - dstart = tstart;*/ - } unsigned int Patch::ChunkSize(Signature signature, @@ -126,3 +109,47 @@ unsigned int Patch::ByteSize(Signature signature, return size; } + + +char *Patch::Compress(unsigned int ram_size, unsigned int &size) { + //TODO use OVERLAP and test speed + //TODO fill chunk padding with zeroes? + //TODO compress only used memory! + size = ram_size + ram_size/64 + 23; + char *buffer = new char[size]; + lzo1x_1_compress(((unsigned char *)start), ram_size, + (unsigned char *)buffer + sizeof(int), &size, + (char *)wrkmem); + + *(int *)buffer = size; + size += sizeof(int); + + + // memcpy(buffer, start, ram_size); + // size = ram_size; + //TODO optimize! + // lzo1x_optimize((unsigned char *)entry.patch->start, + // entry.ram_size * chunk_size, + // compressed, &compressed_size, NULL); + + + return buffer; +} + +void Patch::Decompress(unsigned int ram_size, void *src, unsigned int src_sz) { + + unsigned int size = *(int *)src; + assert(size < src_sz + sizeof(int)); + unsigned int dst_size = ram_size; + + // memcpy(start, src, ram_size); + int ret = lzo1x_decompress_safe(((unsigned char *)src) + sizeof(int), size, + (unsigned char *)start, &dst_size, 0); + if(ret != 0) { + cerr << "Ret from decompress: " << ret << endl; + exit(-1); + } + assert(dst_size == ram_size); + //TODO add 3 to start... so we can use asm_fast decompressor +} + diff --git a/apps/nexus/patch.h b/apps/nexus/patch.h index d8fee324..a3fbaccd 100644 --- a/apps/nexus/patch.h +++ b/apps/nexus/patch.h @@ -49,13 +49,15 @@ class Patch { static unsigned int ByteSize(Signature signature, unsigned short nvert, unsigned short nface); - - - char *start; + + char *Compress(unsigned int ram_size, unsigned int &size); + void Decompress(unsigned int ram_size, void *src, unsigned int src_sz); + unsigned short nv; unsigned short nf; + char *start; float *vstart; //these offset are from vstart! unsigned short cstart; diff --git a/apps/nexus/patchserver.cpp b/apps/nexus/patchserver.cpp index 0b5bf342..52579828 100644 --- a/apps/nexus/patchserver.cpp +++ b/apps/nexus/patchserver.cpp @@ -1,5 +1,4 @@ #include "patchserver.h" - #include #include @@ -8,6 +7,12 @@ using namespace nxs; //TODO support compression! +void outbuffer(unsigned char *b, unsigned int s) { + for(unsigned int i = 0; i < s; i++) + cerr << (unsigned int)b[i]; + cerr << endl; +} + bool PatchServer::Create(const std::string &filename, Signature sig, @@ -47,8 +52,8 @@ bool PatchServer::ReadEntries(FILE *fp) { for(unsigned int i = 0; i < n; i++) { patches[i].patch = NULL; fread(&(patches[i].patch_start), 1, sizeof(unsigned int), fp); - fread(&(patches[i].patch_size), 1, sizeof(unsigned short), fp); - fread(&(patches[i].ram_used), 1, sizeof(unsigned short), fp); + fread(&(patches[i].ram_size), 1, sizeof(unsigned short), fp); + fread(&(patches[i].disk_size), 1, sizeof(unsigned short), fp); patches[i].lru_pos = 0xffffffff; } return true; @@ -59,8 +64,8 @@ bool PatchServer::WriteEntries(FILE *fp) { fwrite(&n, 1, sizeof(int), fp); for(unsigned int i = 0; i < patches.size(); i++) { fwrite(&(patches[i].patch_start), 1, sizeof(unsigned int), fp); - fwrite(&(patches[i].patch_size), 1, sizeof(unsigned short), fp); - fwrite(&(patches[i].ram_used), 1, sizeof(unsigned short), fp); + fwrite(&(patches[i].ram_size), 1, sizeof(unsigned short), fp); + fwrite(&(patches[i].disk_size), 1, sizeof(unsigned short), fp); } return true; } @@ -68,37 +73,64 @@ bool PatchServer::WriteEntries(FILE *fp) { void PatchServer::AddPatch(unsigned short nvert, unsigned short nface) { PatchEntry entry; entry.patch = NULL; - entry.patch_start = Length()/chunk_size; - entry.patch_size = Patch::ChunkSize(signature, nvert, nface, chunk_size); - entry.ram_used = entry.patch_size; + entry.ram_size = Patch::ChunkSize(signature, nvert, nface, chunk_size); + + //if compressed we do not allocate space now (how much anyway?) + // if((signature & NXS_COMPRESSED) != 0) { + entry.disk_size = 0xffff; + entry.patch_start = 0xffffffff; + /* } else { + entry.disk_size = entry.ram_size; + entry.patch_start = Length()/chunk_size; + Redim(Length() + entry.disk_size * chunk_size); + }*/ entry.lru_pos = 0xffffffff; patches.push_back(entry); + - Redim(Length() + entry.patch_size * chunk_size); } Patch &PatchServer::GetPatch(unsigned int idx, unsigned short nvert, unsigned short nface, bool flush) { - assert(idx < patches.size()); PatchEntry &entry = patches[idx]; - if(entry.patch) { + if(entry.patch) { //already on buffer assert(entry.lru_pos < lru.size()); assert(lru[entry.lru_pos].patch == idx); lru[entry.lru_pos].frame = frame++; + } else { - SetPosition(entry.patch_start * chunk_size); + + assert(entry.lru_pos == 0xffffffff); + if(flush) Flush(); + + + char *ram = new char[entry.ram_size * chunk_size]; + entry.patch = new Patch(signature, ram, nvert, nface); - assert(entry.patch_size != 0); - char *start = new char[entry.patch_size * chunk_size]; - ReadBuffer(start, entry.patch_size * chunk_size); - - entry.patch = new Patch(signature, start, nvert, nface); + if(entry.patch_start != 0xffffffff) { //was allocated. + assert(entry.disk_size != 0xffff); + + SetPosition(entry.patch_start * chunk_size); + + if((signature & NXS_COMPRESSED) == 0) { //not compressed + ReadBuffer(ram, entry.disk_size * chunk_size); + } else { + + unsigned char *disk = new unsigned char[entry.disk_size * chunk_size]; + ReadBuffer(disk, entry.disk_size * chunk_size); + + entry.patch->Decompress(entry.ram_size * chunk_size, + disk, entry.disk_size * chunk_size); + delete []disk; + } + } + entry.lru_pos = lru.size(); lru.push_back(PTime(idx, frame++)); - ram_used += entry.ram_used; + ram_used += entry.ram_size; } //avoid frame overflow! @@ -112,27 +144,23 @@ Patch &PatchServer::GetPatch(unsigned int idx, for(unsigned int i = 0; i < lru.size(); i++) patches[lru[i].patch].lru_pos = i; } - - if(flush && ram_used > ram_size * 1.1) - Flush(); - return *(entry.patch); } void PatchServer::Flush() { - cerr << "FLUSHING\n\n\n n"; - cerr << "ram_size: " << ram_size << endl; - cerr << "ram_used: " << ram_used << endl; + + if(ram_used < ram_size * 1.5) return; + make_heap(lru.begin(), lru.end()); + for(unsigned int i = 0; i < lru.size(); i++) + patches[lru[i].patch].lru_pos = i; + while(ram_used > ram_size) { pop_heap(lru.begin(), lru.end()); PTime &ptime = lru.back(); - Flush(ptime.patch); - lru.pop_back(); } - make_heap(lru.begin(), lru.end()); for(unsigned int i = 0; i < lru.size(); i++) patches[lru[i].patch].lru_pos = i; } @@ -146,19 +174,53 @@ void PatchServer::FlushAll() { lru.clear(); } - void PatchServer::Flush(unsigned int patch) { - PatchEntry &entry = patches[patch]; assert(entry.patch); if(!readonly) { //write back patch - SetPosition(entry.patch_start * chunk_size); - WriteBuffer(entry.patch->start, entry.patch_size * chunk_size); + + + if((signature & NXS_COMPRESSED)) { + unsigned int compressed_size; + char *compressed = entry.patch->Compress(entry.ram_size * chunk_size, + compressed_size); + if(entry.disk_size == 0xffff) {//allocate space + assert(entry.patch_start == 0xffffffff); + entry.disk_size = (unsigned int)((compressed_size-1)/chunk_size) + 1; + entry.patch_start = Length()/chunk_size; + Redim(Length() + entry.disk_size * chunk_size); + } else { + cerr << "OOOOPSPPPS not supported!" << endl; + exit(-1); + } + cerr << "Ram size: " << entry.ram_size << endl; + cerr << "Disk size: " << entry.disk_size << endl; + SetPosition(entry.patch_start * chunk_size); + WriteBuffer(compressed, entry.disk_size * chunk_size); + delete []compressed; + } else { + if(entry.disk_size == 0xffff) { + entry.disk_size = entry.ram_size; + entry.patch_start = Length()/chunk_size; + Redim(Length() + entry.disk_size * chunk_size); + } + SetPosition(entry.patch_start * chunk_size); + WriteBuffer(entry.patch->start, entry.disk_size * chunk_size); + } + /* FILE *fo = fopen("tmp", "wb+"); + fwrite(entry.patch->start, 1, entry.disk_size * chunk_size, fo); + fclose(fo); + exit(0);*/ } + delete [](entry.patch->start); delete entry.patch; entry.patch = NULL; entry.lru_pos = 0xffffffff; - ram_used -= entry.ram_used; + ram_used -= entry.ram_size; +} + +void PatchServer::SetRamBufferSize(unsigned int r_buffer) { + ram_size = (unsigned int)(r_buffer/chunk_size) + 1; } diff --git a/apps/nexus/patchserver.h b/apps/nexus/patchserver.h index dd2125bb..5027b690 100644 --- a/apps/nexus/patchserver.h +++ b/apps/nexus/patchserver.h @@ -11,8 +11,8 @@ namespace nxs { struct PatchEntry { Patch *patch; unsigned int patch_start; //granularita' Chunk - unsigned short patch_size; //in chunks - unsigned short ram_used; // in chunks (used when compressed) + unsigned short ram_size; //in chunks + unsigned short disk_size; // in chunks (used when compressed) unsigned int lru_pos; }; @@ -25,7 +25,7 @@ class PatchServer: public File { PTime(unsigned int p = 0xffffffff, unsigned int f = 0xffffffff): patch(p), frame(f) {} - bool operator<(const PTime &p) const { return frame < p.frame; } + bool operator<(const PTime &p) const { return frame > p.frame; } }; @@ -54,6 +54,7 @@ class PatchServer: public File { void Flush(); void FlushAll(); void Flush(unsigned int patch); + void SetRamBufferSize(unsigned int ram_buffer); std::vector patches; std::vector lru; diff --git a/apps/nexus/voronoichain.cpp b/apps/nexus/voronoichain.cpp index da998b91..401854b7 100644 --- a/apps/nexus/voronoichain.cpp +++ b/apps/nexus/voronoichain.cpp @@ -24,6 +24,9 @@ History $Log: not supported by cvs2svn $ +Revision 1.10 2004/10/09 14:46:47 ponchio +Windows porting small changes. + Revision 1.9 2004/10/08 15:12:04 ponchio Working version (maybe) @@ -132,7 +135,7 @@ void VoronoiChain::Init(Crude &crude, float scaling, int steps) { if(fcount[i] > min_size) seeds.push_back(fine[i]); } - swap((std::vector)fine, seeds); + swap(fine, seeds); if(fine.size() == 0) fine.push_back(Point3f(0,0,0)); fine.Init(); @@ -170,7 +173,7 @@ void VoronoiChain::Init(Crude &crude, float scaling, int steps) { if(ccount[i] > (int)min_size) seeds.push_back(coarse[i]); } - swap((std::vector)coarse, seeds); + swap(coarse, seeds); if(coarse.size() == 0) coarse.push_back(Point3f(0,0,0)); coarse.Init(); @@ -404,7 +407,7 @@ void VoronoiChain::BuildLevel(Nexus &nexus, unsigned int offset, if(ccount[i] > (int)min_size) seeds.push_back(coarse[i]); } - swap((vector)coarse, seeds); + swap(coarse, seeds); if(coarse.size() == 0) coarse.push_back(Point3f(0,0,0)); coarse.Init(); diff --git a/apps/nexus/voronoinxs.cpp b/apps/nexus/voronoinxs.cpp index 8c54d026..b870abec 100644 --- a/apps/nexus/voronoinxs.cpp +++ b/apps/nexus/voronoinxs.cpp @@ -24,6 +24,9 @@ History $Log: not supported by cvs2svn $ +Revision 1.13 2004/10/09 17:32:25 ponchio +Ram buffer option added (last) + Revision 1.12 2004/10/09 17:29:04 ponchio Ram buffer option added (again) @@ -131,9 +134,10 @@ int main(int argc, char *argv[]) { unsigned int max_level = 0xffffffff; float scaling = 0.5; unsigned int ram_buffer = 128000000; + unsigned int chunk_size = 1024; int option; - while((option = getopt(argc, argv, "f:t:l:s:d:ro:b:")) != EOF) { + while((option = getopt(argc, argv, "f:t:l:s:d:ro:b:c:")) != EOF) { switch(option) { case 'f': patch_size = atoi(optarg); if(patch_size == 0 || patch_size > 32000) { @@ -171,7 +175,18 @@ int main(int argc, char *argv[]) { break; case 'r': stop_after_remap = true; break; case 'o': optimization_steps = atoi(optarg); break; - case 'b': ram_buffer = atoi(optarg); break; + case 'b': ram_buffer = atoi(optarg); + if(ram_buffer == 0) { + cerr << "Invalid ram buffer: " << optarg << endl; + return -1; + } + break; + case 'c': chunk_size = atoi(optarg); + if(chunk_size == 0) { + cerr << "Invalid chunk size: " << optarg << endl; + return -1; + } + break; default: cerr << "Unknown option: " << (char)option << endl; return -1; } @@ -206,11 +221,12 @@ int main(int argc, char *argv[]) { string output = argv[optind+1]; Nexus nexus; - if(!nexus.Create(output, NXS_FACES)) { + nexus.patches.SetRamBufferSize(ram_buffer); + if(!nexus.Create(output, NXS_FACES, chunk_size)) { cerr << "Could not create nexus output: " << output << endl; return -1; } - nexus.patches.ram_size = ram_buffer; + if(patch_threshold == 0xffffffff) patch_threshold = patch_size/4;