From 2d244d6326615b12d12a10578493b0fd354fab80 Mon Sep 17 00:00:00 2001 From: andrea matte <30327952+andreamatt@users.noreply.github.com> Date: Mon, 1 Jan 2018 20:21:41 +0100 Subject: [PATCH 1/5] Added extract(destination) method - added extract(destinationDir) method - commented some code - removed require('path') (never used) - added loaded (boolean) property - added a few more exeptions handlers - commented MAX_PATH (never used) --- index.js | 95 ++++++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 79 insertions(+), 16 deletions(-) diff --git a/index.js b/index.js index 17ba42e..3816b00 100644 --- a/index.js +++ b/index.js @@ -1,12 +1,12 @@ -"use strict"; +'use strict'; var crc = require('crc'); var fs = require('fs'); var jBinary = require('jbinary'); -var path = require('path'); let TYPESET = { - 'jBinary.littleEndian': true, + 'jBinary.littleEndian': true, + vpkHeader: jBinary.Type({ read: function() { let header = {}; @@ -32,15 +32,16 @@ let TYPESET = { return header; } - }), + }), + vpkDirectoryEntry: jBinary.Type({ read: function() { let entry = this.binary.read({ - crc: 'uint32', - preloadBytes: 'uint16', - archiveIndex: 'uint16', - entryOffset: 'uint32', - entryLength: 'uint32' + crc: 'uint32', // crc integrity + preloadBytes: 'uint16', // size of preload (almost always 0) (used for small but critical files) + archiveIndex: 'uint16', // on which archive the data is stored (7fff means on _dir archive) + entryOffset: 'uint32', // if on _dir, this is offset of data from dirEntry end. If on other archive, offset from start of it + entryLength: 'uint32' // size of data }); let terminator = this.binary.read('uint16'); @@ -50,7 +51,8 @@ let TYPESET = { return entry; } - }), + }), + vpkTree: jBinary.Type({ read: function() { let files = {}; @@ -102,14 +104,16 @@ let TYPESET = { }) }; +// header size in bytes let HEADER_1_LENGTH = 12; let HEADER_2_LENGTH = 28; -let MAX_PATH = 260; +// let MAX_PATH = 260; class VPK { constructor(path) { - this.directoryPath = path; + this.directoryPath = path; + this.loaded = false; } isValid() { @@ -123,7 +127,7 @@ class VPK { return true; } - catch (e) { + catch (error) { return false; } } @@ -131,8 +135,13 @@ class VPK { load() { let binary = new jBinary(fs.readFileSync(this.directoryPath), TYPESET); - this.header = binary.read('vpkHeader'); - this.tree = binary.read('vpkTree'); + try{ + this.header = binary.read('vpkHeader'); + this.tree = binary.read('vpkTree'); + this.loaded = true; + } catch(error) { + throw new Error('Failed loading ' + this.directoryPath); + } } get files() { @@ -168,6 +177,7 @@ class VPK { fs.readSync(directoryFile, file, entry.preloadBytes, entry.entryLength, offset + entry.entryOffset); } else { + // read from specified archive let fileIndex = ('000' + entry.archiveIndex).slice(-3); let archivePath = this.directoryPath.replace(/_dir\.vpk$/, '_' + fileIndex + '.vpk'); @@ -181,7 +191,60 @@ class VPK { } return file; - } + } + + extract(destinationDir) { + // if not loaded yet, load it + if(this.loaded === false){ + try { + this.load(); + } catch (error) { + throw new Error('VPK was not loaded and it failed loading'); + } + } + + var failed = []; + // make sure destinationDir exists + try { + fs.ensureDirSync(destinationDir); + } catch (error) { + throw new Error('Destination dir cant be ensured'); + } + + // extract files one by one + for (var file of this.files) { + // destination of this file (with file name and extension) + var destFile = destinationDir + '/' + file; + // destination of this file (only the directory) + var fileDestDir = destFile.substr(0, destFile.lastIndexOf('/')); + + // make sure destination dir of this file exists + try { + fs.ensureDirSync(fileDestDir); + } catch (error) { + throw new Error('Error ensuring file directory: ' + fileDestDir); + } + + // get the file + try { + var fileBuffer = this.getFile(file); + } catch (error) { + throw error; + } + + // write it + try { + fs.writeFileSync(destFile, fileBuffer); + } catch (error) { + failed.push(destFile); + } + } + + // throw all failed files + if (failed.length !== 0) { + throw new Error('Failed extrating following files: \r\n' + failed.toString()); + } + } } module.exports = VPK; From 053b31fa99742d8458b7ff978626aecdfe3c312b Mon Sep 17 00:00:00 2001 From: andrea matte <30327952+andreamatt@users.noreply.github.com> Date: Tue, 2 Jan 2018 12:01:50 +0100 Subject: [PATCH 2/5] using fs-extra instead of fs --- index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/index.js b/index.js index 3816b00..cf188a6 100644 --- a/index.js +++ b/index.js @@ -1,7 +1,7 @@ 'use strict'; var crc = require('crc'); -var fs = require('fs'); +var fs = require('fs-extra'); var jBinary = require('jbinary'); let TYPESET = { From 305173d67eee47fd8dd248e24709b04a65a86d77 Mon Sep 17 00:00:00 2001 From: andrea matte <30327952+andreamatt@users.noreply.github.com> Date: Fri, 5 Jan 2018 23:45:04 +0100 Subject: [PATCH 3/5] Now able to create v1 VPKs - added VPKcreator class: use load and write to create a vpk file of a directory (ONLY FOR V1 VPK) - now exports { VPK, VPKcreator } --- index.js | 132 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 129 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index cf188a6..2b91be3 100644 --- a/index.js +++ b/index.js @@ -40,7 +40,7 @@ let TYPESET = { crc: 'uint32', // crc integrity preloadBytes: 'uint16', // size of preload (almost always 0) (used for small but critical files) archiveIndex: 'uint16', // on which archive the data is stored (7fff means on _dir archive) - entryOffset: 'uint32', // if on _dir, this is offset of data from dirEntry end. If on other archive, offset from start of it + entryOffset: 'uint32', // if on _dir, this is offset of data from tree end. If on other archive, offset from start of it entryLength: 'uint32' // size of data }); @@ -89,7 +89,7 @@ let TYPESET = { fullPath = directory + '/' + fullPath; } - let entry = this.binary.read('vpkDirectoryEntry'); + let entry = this.binary.read('vpkDirectoryEntry'); entry.preloadOffset = this.binary.tell(); this.binary.skip(entry.preloadBytes); @@ -247,4 +247,130 @@ class VPK { } } -module.exports = VPK; +function listFiles(directory, first = true, root = directory){ + var files = []; + var filesHere = fs.readdirSync(directory); + filesHere.forEach(file => { + if(fs.statSync(directory + "/" + file).isDirectory()){ + files = files.concat(listFiles(directory + "/" + file, false, root)); + } + else{ + files.push({ + location: first ? " " : directory.substr(root.length + 1), + name: file.substr(0, file.lastIndexOf(".")), + extension: file.substr(file.lastIndexOf(".") + 1), + crc: crc.crc32(fs.readFileSync(directory + "/" + file)), + dataSize: fs.statSync(directory + "/" + file).size, + fullPath: directory + "/" + file + }); + } + }); + + return files; +} + +class VPKcreator { + constructor(directory){ + this.root = directory; + this.loaded = false; + } + + isValid(){ + try { + if(fs.statSync(this.root).isDirectory()){ + return true; + } + return false; + } catch (error) { + return false; + } + } + + // load dir as vpk (only supports VPK 1 atm) + load(version = 1){ + if(version != 1){ + throw new Error("Version not supported"); + } + if(this.isValid()){ + + // create all entries + var files = listFiles(this.root); + + // calculate total size of entries + var totalSize = 0; + totalSize += 4; // signature + totalSize += 4; // vpk version + totalSize += 4; // treeSize + var treeSize = 0; + this.totalData = 0; + files.forEach(file => { + let entrySize = 0; + entrySize = (file.location.length +1) + (file.name.length +1) + (file.extension.length +1); + entrySize += 4; // crc + entrySize += 2; // preloadBytes + entrySize += 2; // archiveIndex + entrySize += 4; // entryOffset + entrySize += 4; // entryLength + entrySize += 2; // terminator + entrySize += 2; // 2 nulls terminating + file.entrySize = entrySize; + treeSize += entrySize; + this.totalData += file.dataSize; + }); + treeSize += 1; // the last null + totalSize += treeSize; + + // set all offsets + var offset = 0; // offset from tree end + files.forEach(file => { + file.dataOffset = offset; + offset += file.dataSize; + }); + + this.totalSize = totalSize; + this.treeSize = treeSize; + this.tree = files; + + this.loaded = true; + } + } + + save(destinationFile){ + // header + var header = new Buffer(HEADER_1_LENGTH); + header.writeUInt32LE(0x55aa1234, 0); + header.writeUInt32LE(1, 4); + header.writeUInt32LE(this.treeSize, 8); + + // tree + var tree = new Buffer(this.treeSize); + var offset = 0; + this.tree.forEach(file => { + console.log(file); + tree.write(file.extension + "\0" + file.location + "\0" + file.name + "\0", offset); + let relOff = offset + file.entrySize - 20; + tree.writeUInt32LE(file.crc, relOff); // crc + tree.writeUInt16LE(0x0000, relOff +4); // preloadByes + tree.writeUInt16LE(0x7fff, relOff +6); // archiveIndex + tree.writeUInt32LE(file.dataOffset, relOff +8); // entryOffset + tree.writeUInt32LE(file.dataSize, relOff +12); // entryLength + tree.writeUInt16LE(0xffff, relOff +16); // terminator + tree.write("\0\0", relOff +18); // 2 nulls + + offset += file.entrySize; + }); + tree.write("\0", this.treeSize); // last null + + fs.writeFileSync(destinationFile, Buffer.concat([header, tree])); + + // data + var data = new Buffer(this.totalData); + this.tree.forEach(file => { + console.log(file); + let fileData = fs.readFileSync(file.fullPath); + fs.appendFileSync(destinationFile, fileData); + }); + } +} + +module.exports = {VPK, VPKcreator}; From a7eb050e7a37ad448b44e7ead25d2224a82fadab Mon Sep 17 00:00:00 2001 From: andrea matte <30327952+andreamatt@users.noreply.github.com> Date: Sat, 6 Jan 2018 00:38:50 +0100 Subject: [PATCH 4/5] updated readme --- README.md | 44 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 43 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 32513fb..04437a2 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,44 @@ # node-vpk -parser and extractor for the Valve Pack Format +extractor and creator for the Valve Pack Format + +### Prerequisites + +Requires fs-extra and jBinary, +but npm will install them automatically if you follow my instructions + +### Installing +globally +``` +npm install -g vpk +``` +or locally +``` +npm install vpk +``` +(Those will also install fs-extra and jBinary, no need to worry about them) +### How to use + +To extract a V1/V2 VPK: +``` +const {VPK} = require("node-vpk"); + +// load a vpk (V1/V2) (ALWAYS select the _dir file) +var my_vpk = new VPK("C:/Programs("C:/Program Files (x86)/Steam/steamapps/common/dota 2 beta/game/dota/pak01_dir"); +my_vpk.load(); + +// extract it +my_vpk.extract("C:/Users/Public/Desktop/pak01_dir"); +``` + +To create a V1 vpk (V2 coming soon): +``` +const {VPKcreator} = require("node-vpk"); + +// load a directory +var my_vpk = new VPKcreator("C:/Users/Public/Desktop/pak01_dir"); +my_vpk.load(); + +// save it as .vpk +my_vpk.save("C:/Users/Public/Desktop/my_created_vpk.vpk"); +``` + From bb39bb6cde378516b9c5ff8ff380f7df5cc14552 Mon Sep 17 00:00:00 2001 From: andrea matte <30327952+andreamatt@users.noreply.github.com> Date: Sat, 6 Jan 2018 01:15:00 +0100 Subject: [PATCH 5/5] fixed dependencies --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 04437a2..ba5201f 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ extractor and creator for the Valve Pack Format ### Prerequisites -Requires fs-extra and jBinary, +Requires fs-extra, jBinary and crc, but npm will install them automatically if you follow my instructions ### Installing @@ -15,7 +15,7 @@ or locally ``` npm install vpk ``` -(Those will also install fs-extra and jBinary, no need to worry about them) +(Those will also install dependencies, no need to worry about them) ### How to use To extract a V1/V2 VPK: