Now we need to recover (literally decompress) all the files from the hex datastream to their original state.
Following the guidelines in the previously mentioned manual (link), we will use java.util.zip.Inflater class. However, the tricky part is that the manual does not describe the header structure for segmented files.
According to the manual, if the RC type indicator equals 0x2D, the next byte indicates the length of compressed data. Similarly, if the RC type indicator equals 0x2E, the next 2 bytes indicate the compressed data length. However, if the RC type indicator equals 0x2E and the data is segmented, the next 2 bytes are 0xFFFF, and the following 4 bytes indicate the compressed data length. This was a pitfall I initially overlooked and had some fun with debugging thereafter.
Eventually, using ChatGPT, I managed to create a script that takes the file we created in the first part of this article as an input argument and recovers all the compressed files in the working directory.
A few notes on running the script: I explicitly provided the path to the “opencsv” and “commons-lang” JARs when executing the script. Additionally, I used the options “-Xmx1024m – Xms256m” to set the maximum heap size to 1024 MB and the initial heap size to 256 MB, allowing the script to process large files.
import com.opencsv.*;
import com.opencsv.exceptions.*;
import java.io.*;
import java.util.List;
import java.util.Objects;
import java.util.zip.*;
public class Main {
public static byte[] hexStringToByteArray(String input) {
int len = input.length();
if (len == 0) return new byte[]{};
byte[] data;
int startIdx;
if (len % 2 != 0) {
data = new byte[(len / 2) + 1];
data[0] = (byte) Character.digit(input.charAt(0), 16);
startIdx = 1;
} else {
data = new byte[len / 2];
startIdx = 0;
}
for (int i = startIdx; i < len; i += 2) {
data[(i + 1) / 2] = (byte) ((Character.digit(input.charAt(i), 16) << 4) + (Character.digit(input.charAt(i+1), 16)));
}
return data;
}
//writing decompressed data on disk
public static void WriteToFile(String Name1, String Name2, byte[] text_to_write) {
//Path currentRelativePath = Paths.get("");
//String working_directory = currentRelativePath.toAbsolutePath().toString();
String working_directory = "/Users/0xMegaworm/IdeaProjects/untitled/Tests_results";
try (OutputStream os = new FileOutputStream(new File(working_directory + "/" + Name1 + "_" + Name2))) {
os.write(text_to_write);
System.out.println("File " + Name1 + "_" + Name2 + " created");
} catch (IOException e) {
//noinspection CallToPrintStackTrace
e.printStackTrace();
}
}
//get compressed data start position and length from header
public static int[] GetCompressedLength(byte[] compressed_data_hex) {
int[] compressed_length_and_start = new int[2];
// skip prefix: _RCFM*=
// read starting from the 8th byte, which means the next 1 byte is the length
if (compressed_data_hex[7] == 0x2D)
{
compressed_length_and_start[0] = 10;
compressed_length_and_start[1] = compressed_data_hex[8]&0xFF;
}
else if ((compressed_data_hex[7] == 0x2E)&((compressed_data_hex[8]&0xFF) == 0xFF)&((compressed_data_hex[9]&0xFF) == 0xFF))
{
// which means the next 2 bytes are padding and then 4 bytes are the length
compressed_length_and_start[0] = 15;
compressed_length_and_start[1] = ((((((compressed_data_hex[10]&0xFF)*256) +(compressed_data_hex[11]&0xFF))*256)+(compressed_data_hex[12]&0xFF))*256)+(compressed_data_hex[13]&0xFF);
}
else {
// which means the next 2 bytes are the length
compressed_length_and_start[0] = 11;
compressed_length_and_start[1] = (compressed_data_hex[8]&0xFF);
compressed_length_and_start[1] = compressed_length_and_start[1]*256;
compressed_length_and_start[1] = compressed_length_and_start[1]+(compressed_data_hex[9]&0xFF);
}
return compressed_length_and_start;
}
public static byte[] Inflater_Method(byte[] compressed_data_hex_w_o_headers, String original_size) throws DataFormatException {
///////////////////////////////////////////////////////////////////////
// use Inflater to get back the original data
// Inflater
Inflater i = new Inflater();
// set the input for inflator
//i.setInput(output);
i.setInput(compressed_data_hex_w_o_headers);
// output bytes
byte[] inflater_output = new byte[Integer.parseInt(original_size.substring(0,original_size.indexOf(".")))];
// decompress the data
int decompressed_size = i.inflate(inflater_output);
//check for successful decompression
if (decompressed_size == 0) {
System.out.println("Error: Nothing was decompressed");
return null;
}
else if (decompressed_size != Integer.parseInt(original_size.substring(0,original_size.indexOf(".")))){
System.out.println("Warning: Something went wrong with decompression of file. "+decompressed_size+" bytes decompressed, should be "+original_size);
//return null; may occure for segmented files?
}
else {
System.out.println("Inflater finished decompression: "+original_size+" bytes decompressed");
}
i.end();
return inflater_output;
}
public static void PrintGeneralInfoOfFile(String[] input) {
System.out.println("CHG: "+ input[0]); ///CHG ID
System.out.println("Compressed file name: "+input[1]); ///File Name
if (input[3].length() <100) System.out.println("Compressed data: " + input[3]);
else System.out.println("Compressed data: " + input[3].substring(0, 30) + "...");
}
public static byte[] PrepareCompressedDataToInflater(String[] input){
try {
int compressed_length_input = Integer.parseInt(input[4].substring(0, input[4].indexOf("."))); ///Compressed data length from csv
System.out.println("compressed data length input: " + compressed_length_input + " bytes");
} catch (NumberFormatException e) {
System.out.println("Error: Compressed length error: compressed length input = 0... SKIPPING THIS ROW");
return null;
}
String compressed_data = input[3]; ///Whole Header+Size+Data blob
byte[] compressed_data_hex = hexStringToByteArray(compressed_data);
int length = compressed_data.length(); //whole blob length
if ( length < 10 ) {
System.out.println ("Error: Incorrect length of compressed data according to Microfocus manual...SKIPPING THIS ROW");
return null; // error
}
//get compressed data start position and length from header
int[] compressed_length_and_start = GetCompressedLength(compressed_data_hex);
if (compressed_length_and_start == null) return null;
//get only compressed data from the full header+size+data blob
byte[] compressed_data_hex_w_o_headers = new byte[compressed_length_and_start[1]];
System.arraycopy(compressed_data_hex, compressed_length_and_start[0]-1, compressed_data_hex_w_o_headers, 0, compressed_length_and_start[1]);
return compressed_data_hex_w_o_headers;
}
public static void DecToHexStreamOnDisk(byte[] compressed_data_hex_w_o_headers) {
//write compressed_data_hex_w_o_headers to file for debug
//
String outputFile = "/Users/0xMegaworm/Downloads/debug_data_hex.txt";
try (OutputStream outputStream = new BufferedOutputStream(new FileOutputStream(outputFile, true))) {
StringBuilder hexStream = new StringBuilder();
for (byte compressed_data_hex_w_o_header : compressed_data_hex_w_o_headers) {
String hexString = Integer.toHexString(compressed_data_hex_w_o_header&0xFF);
if (hexString.length() < 2) {
hexString = "0" + hexString;
}
hexStream.append(hexString);
}
hexStream.append("\n");
outputStream.write(hexStream.toString().getBytes());
System.out.println("Hex stream successfully written to the file");
} catch (IOException e) {
e.printStackTrace();
}
}
public static byte[] concatenateByteArrays(byte[] a, byte[] b) {
byte[] result = new byte[a.length + b.length];
System.arraycopy(a, 0, result, 0, a.length);
System.arraycopy(b, 0, result, a.length, b.length);
return result;
}
public static int ParseCsv(List<String[]> rows, int currentIndex, int segments) throws UnsupportedEncodingException, DataFormatException {
String[] currentRow = rows.get(currentIndex);
PrintGeneralInfoOfFile(currentRow);
byte[] compressed_data_hex_w_o_headers = new byte[0];
for (int i=0; i < segments + 1; i++) {
byte[] next_data_segment = PrepareCompressedDataToInflater(rows.get(currentIndex + i));
compressed_data_hex_w_o_headers = concatenateByteArrays(compressed_data_hex_w_o_headers, Objects.requireNonNull(next_data_segment));
//DecToHexStreamOnDisk(next_data_segment);
//DecToHexStreamOnDisk(compressed_data_hex_w_o_headers);
}
//decompress compressed data
byte[] inflater_output;
inflater_output = Inflater_Method(compressed_data_hex_w_o_headers, currentRow[5]);
//writing decompressed data on disk if decompressed successfully
if (inflater_output != null) WriteToFile(currentRow[0], currentRow[1], inflater_output);
else return 0;
return 1;
}
public static boolean hasNextSegment(List<String[]> rows, int currentIndex) {
if (currentIndex + 1 < rows.size()) {
String[] currentRow = rows.get(currentIndex);
String[] nextRow = rows.get(currentIndex + 1);
// Check if CHG Name and File Name are the same in the next row
return currentRow[0].equals(nextRow[0]) && currentRow[1].equals(nextRow[1]) &&(Integer.parseInt(nextRow[2].substring(0,nextRow[2].indexOf(".")))!=0);
}
return false;
}
public static int CountSegments(List<String[]> rows, int currentIndex) {
// if the compressed file is segmented returns number of segments, otherwise - 0
int segments = 0;
while (hasNextSegment(rows, currentIndex)) {
segments++;
currentIndex++;
}
return segments;
}
public static void main(String[] args) throws IOException, CsvException {
if (args.length == 0) {
System.err.println("Missing arguments"+System.lineSeparator()+"Try '--help' for more information"+System.lineSeparator()+"Try '--version' for version information");
}
else if (args[0].equalsIgnoreCase("--help")) {
System.out.println("Usage: java OpenCsvExample [FILE]"+System.lineSeparator()+"FILE is the .csv without headers with the following content and fields order:"+System.lineSeparator()+"CHG Name | File Name | Segment | Data | Compressed Size | Original Size | other optional fields...");
}
else if (args[0].equalsIgnoreCase("--version")) {
System.out.println("OpenCsvExample version 1.0"+System.lineSeparator()+""+System.lineSeparator()+"IT Audit"+System.lineSeparator()+"");
}
else {
String fileName = args[0];
try (CSVReader reader = new CSVReader(new FileReader(fileName))) {
List<String[]> r = reader.readAll();
for (int i = 0; i < r.size(); i++) {
boolean isSegmented = hasNextSegment(r, i);
int segments = CountSegments(r,i);
try {
if (isSegmented) System.out.println("Segmented file detected for row number " + (i + 1));
if (ParseCsv(r, i, segments) != 0) {
System.out.println("Successfully processed row number " + (i + 1) + System.lineSeparator());
}
else {
System.out.println("Error: Error in row " + (i + 1) + System.lineSeparator());
}
} catch (UnsupportedEncodingException | DataFormatException e) {
e.printStackTrace();
}
i = i + segments; //since if there are no segments, the segments var will equal 0
}
}
}
}
}
This is the second part of the article. The first part can be found here (link)
Leave a Reply