import java.io.IOException;

import java.io.sstream;

import java.nio.file.Files;

import java.nio.file.Path;

import java.nio.file.Paths;

import java.util.ArrayList;

import java.util.List;

import java.util.regex.Matcher;

import java.util.regex.Pattern;

public class BinaryTokenizer {

public static List<String> tokenizeBinaryFile(Path filePath) throws IOException {

List<String> tokens = new ArrayList<>();

String fileContent = new String(Files.readAllBytes(filePath)); // Read entire file content

// Define regular expressions for different token types

Pattern commentPattern = Pattern.compile("(?m)^(//.*)", Pattern.MULTILINE); // Comments starting with //

Pattern whitespacePattern = Pattern.compile("\\s+"); // Whitespace

Pattern identifierPattern = Pattern.compile("([a-zA-Z_][a-zA-Z0-9_]*)[\\s\\.,;:]?"); // Identifiers

Pattern numberPattern = Pattern.compile("[-+]?\\d+(\\.\\d+)?"); // Numbers

Matcher commentMatcher = commentPattern.matcher(fileContent);

Matcher whitespaceMatcher = whitespacePattern.matcher(fileContent);

Matcher identifierMatcher = identifierPattern.matcher(fileContent);

Matcher numberMatcher = numberPattern.matcher(fileContent);

// Extract tokens

while (commentMatcher.find()) {

tokens.add("COMMENT:" + commentMatcher.group()); // Add comment token

}

while (whitespaceMatcher.find()) {

tokens.add("WHITESPACE:" + whitespaceMatcher.group()); // Add whitespace token

}

while (identifierMatcher.find()) {

tokens.add("IDENTIFIER:" + identifierMatcher.group()); // Add identifier token

}

while (numberMatcher.find()) {

tokens.add("NUMBER:" + numberMatcher.group()); // Add number token

}

return tokens;

}

public static void main(String[] args) throws IOException {

// Example usage:

Path filePath = Paths.get("example.bin"); // Replace with your binary file path

// Create a dummy binary file for testing

try {

Files.write(filePath, "int x = 10; // This is a comment\nfloat y = 3.14; \n int z = -5;".getBytes());

} catch (IOException e) {

System.err.println("Error creating dummy file: " + e.getMessage());

return;

}

List<String> tokens = tokenizeBinaryFile(filePath);

for (String token : tokens) {

System.out.println(token); // Print the tokens

}

}

}